none
Adding char to specific line in text file RRS feed

  • Question

  • Hi,

    I have weather station text files whose data need validation.   There are 17,000+ station files; each station has 42,368 lines.  So I'm developing a VB.Net console application that creates a text file listing all the errors in the data.

    Using this error file, I need to read the corresponding weather station text file and fix all errors in the weather station files.  However, I'm getting an error, "The process cannot access the weather station file because it's being used by another process."  Below is my code snippet:

    Module Module 1
    	Dim stationPath As String = "E:\EpicWeather" ' location of epic files
        	Dim errorPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop)  ' error path
        	Dim errors As ArrayList = New ArrayList()
        	Dim errorFile = errorPath & "\" & "EpicFormatError.txt"
    
    Sub Main()
    	ValidateFormat(stationPath)
    
    	CorrectErrors(errorFile)
    
    End Sub
    
    Private Sub ValidateFormat(stationPath As String)
    	stationPath = "F:\Projects2015VB\ValidateStationData\EpicWeather"
            Dim files = Directory.GetFiles(stationPath)
            Dim yyyy = "", mm = "", dd = ""
    
            Dim fileCount = 0
            For Each file In files
                Dim lines() As String = Nothing
                Dim totalLines = 0
                Dim reader As FileIO.TextFieldParser = New FileIO.TextFieldParser(file)
                reader.TextFieldType = FileIO.FieldType.FixedWidth
                Dim day = 0
    
                Dim station = Path.GetFileNameWithoutExtension(file)
    
                'Count lines
                lines = New StreamReader(file).ReadToEnd().Split(vbCrLf)        'totalLines = 42369
                totalLines = lines.GetLength(0)
    
                'Set reader's width based on format columns
                reader.SetFieldWidths(6, 4, 4, 24)
    
                Dim lineCount = 0
    
                While Not reader.EndOfData
                    Try
                        For Each line In lines
                            'Read one line at a time
                            Dim currField = reader.ReadFields()     ' 4 columns
    
                            If currField IsNot Nothing Then
                                yyyy = currField(0)
                                mm = currField(1)
                                dd = currField(2)
    
                                If CInt(dd) <= 31 Then
                                    day = CInt(dd) + 1
                                End If
    
                                'Add last line to error array
                                If line.Length < 38 Then
                                    Dim err = "Format Error:  " & yyyy & " " & mm & " " & day & "  Station  " & station
                                    err = err.Replace(vbLf, " ")
                                    errors.Add(err)
                                End If
                            End If
    
                            lineCount = lineCount + 1
                        Next
    
                    Catch ex As FileIO.MalformedLineException
                        'Add lines to error array
                        Dim err = "Format Error:  " & yyyy & " " & mm & " " & day & "  Station  " & station
                        err = err.Replace(vbLf, " ")
                        errors.Add(err)
    
                        Dim msg = ex.Message & yyyy & mm & dd & "  " & station
                        msg = msg.Replace(vbLf, "  ")
                        Console.WriteLine(msg)
                    End Try
    
    
                End While
    
                reader.Dispose()
                fileCount += 1
            Next
        End Sub
    
        Private Sub CorrectErrors(errorFile As String)
            Dim errResult() As String = Nothing
            Dim weatherResult() As String = Nothing
            Dim station As String = ""
            Dim totalLines As Integer = 0
            Dim yr = 0, mo = 0, dy = 0
            Dim yyyy = 0, mm = 0, dd = 0
            Dim filePath As String = ""
            Dim errDate, weatherDate As Date
            Dim sol, tmax, tmin, pcp As Single
    
            Dim count = 0
            Using reader As New FileIO.TextFieldParser(errorFile)
                reader.TextFieldType = FileIO.FieldType.FixedWidth
                reader.SetFieldWidths(6, 7, 6, 3, 3, 9, 13)
    
                errResult = New StreamReader(errorFile).ReadToEnd().Split(vbCrLf)
                totalLines = errResult.GetLength(0)
    
                For i = 0 To totalLines - 1
                    If Not i = totalLines - 1 Then
                        errResult(i) = errResult(i).Replace(vbLf, "")
                    End If
                Next
    
                Do While Not reader.EndOfData
                    Dim errRow = reader.ReadFields()
    
                    'Get date and station name
                    yr = CInt(errRow(2))
                    mo = CInt(errRow(3))
                    dy = CInt(errRow(4))
                    errDate = New DateTime(yr, mo, dy)
                    station = errRow(errRow.Length - 1)
    
                    'Get station file
                    filePath = stationPath & "\" & station & ".txt"
    
                    Using wReader As New FileIO.TextFieldParser(filePath)
                        wReader.TextFieldType = FileIO.FieldType.FixedWidth
                        wReader.SetFieldWidths(6, 6, 4, 4, 24)
                        weatherResult = New StreamReader(filePath).ReadToEnd().Split(vbCrLf)
    
                        For i = 0 To weatherResult.Length - 1
                            If Not i = weatherResult.Length - 1 Then
                                weatherResult(i) = weatherResult(i).Replace(vbLf, "")
                            End If
                        Next
    
                        For j = 0 To weatherResult.Length - 1
                            yyyy = Mid(weatherResult(j), 1, 6)
                            mm = Mid(weatherResult(j), 7, 4)
                            dd = Mid(weatherResult(j), 11, 4)
    
                            weatherDate = New DateTime(yyyy, mm, dd)
    
                            If errDate = weatherDate And weatherResult(j).Length < 38 And Not weatherResult(j).Contains("-") Then
                                sol = Mid(weatherResult(j), 15, 6)
                                tmax = Mid(weatherResult(j), 21, 6)
                                tmin = Mid(weatherResult(j), 27, 6)
                                pcp = Mid(weatherResult(j), 33, 6)
    
                                Dim newStr As String = Mid(weatherResult(j), 17, 24)
                                newStr = newStr & "0"
                                Dim writer = My.Computer.FileSystem.OpenTextFileWriter(filePath, False, System.Text.Encoding.ASCII)   'Error here*************
                                writer.WriteLine(newStr)
                            End If
                        Next
    
    
                    End Using
                Loop
    
                count = count + 1
            End Using
        End Sub

    Sample Error Text File:

    Format Error:  1900 03 24  Station  US10cumi003
    Format Error:  1917 10 20  Station  US10cumi003
    Format Error:  1923 05 8  Station  US10cumi003
    Format Error:  1926 10 23  Station  US10cumi003
    Format Error:  1935 02 15  Station  US10cumi003
    Format Error:  1937 03 7  Station  US10cumi003
    Format Error:  1944 11 12  Station  US10cumi003
    Format Error:  1954 11 15  Station  US10cumi003
    Format Error:  1976 11 3  Station  US10cumi003
    Format Error:  1986 03 27  Station  US10cumi003
    Format Error:  2013 01 6  Station  US10cumi003

    Sample Weather Station text file:

      1900  03  22   0.0020.64-00.79000.00
      1900  03  23   0.0016.64-00.48000.00
      1900  03  24   0.0007.8100.00004.48
      1900  03  25   0.0011.80000.48004.26
      1900  03  26   0.0010.92-02.09000.00
      1917  10  18   0.0010.09000.39000.07
      1917  10  19   0.0008.26-10.39000.01
      1917  10  20   0.0008.0500.00000.57
      1917  10  21   0.0015.90-05.04000.00
      1917  10  22   0.0012.02-00.91000.01
      1986  03  26   0.0015.82002.15000.00
      1986  03  27   0.0015.7800.00000.00
      1986  03  28   0.0025.82005.02000.00
      2013  01  05   0.0-00.18-11.86000.00
      2013  01  06   0.000.00-13.83000.00
      2013  01  07   0.0000.54-12.43000.00


    Marilyn Gambone

    Tuesday, June 18, 2019 8:11 PM

Answers

  • Hi

    I suspect I am wrong, but, here is code that I *think* does what you describe in your last post. It will cycle through all the files in the path and change the lines to 38 length if less than that, and will write the changes back to original file. Will also write a Log file to the desktop showing changes.

    As I say, I imagine your requirements stretch to more than that, but I can only go on information available.

    The code below is 'short'. It is basically one Sub, one variable and a change ro the calling context to comment out original and add new.

    Option Strict On
    Option Explicit On
    Module Module1
    	Dim stationPath As String = "C:\Users\lesha\Desktop\EpicWeather"
    	' location of epic files
    	Dim errorPath As String = Environment.GetFolderPath(Environment.SpecialFolder.Desktop)
    	' error path
    	Dim errors As ArrayList = New ArrayList()
    	Dim errorFile As String = errorPath & "\" & "EpicFormatError.txt"
    	Sub Main()
    		'	ValidateFormat(stationPath)
    		'	CorrectErrors(errorFile)
    
    		CheckLineLengths()
    	End Sub
    	Dim log As New List(Of String)
    	Private Sub CheckLineLengths()
    		log.Clear()
    		Dim changes As Integer = 0
    		Dim files() As String = IO.Directory.GetFiles(stationPath)
    		For Each file In files
    			Dim lines() As String = IO.File.ReadAllLines(file)
    			Dim amended As Boolean = False
    			For line As Integer = 0 To lines.Count - 1
    				While lines(line).Length < 38
    					log.Add(file)
    					log.Add(lines(line))
    
    					lines(line) = lines(line) & "0"
    					changes += 1
    					log.Add(lines(line))
    					log.Add("-----------")
    
    					amended = True
    				End While
    			Next
    			If amended Then
    				IO.File.WriteAllLines(file, lines)
    				log.Add("Total changes = " & changes.ToString)
    				IO.File.WriteAllLines(errorFile, log.ToArray)
    			End If
    		Next
    	End Sub
    	End Class


    Regards Les, Livingston, Scotland



    • Edited by leshay Wednesday, June 19, 2019 5:17 PM
    • Marked as answer by deskcheck1 Thursday, June 20, 2019 11:48 AM
    Wednesday, June 19, 2019 5:15 PM

All replies

  • Hi

    A brief look through ypur code would suggest you are trying to access the same file via a TextFieldParser and also a StreamReader at the same time. This would give a similar error to the one you are seeing.

    Is the error being thrown by the line containing the StreamReader declaration?

    Regards Les, Livingston, Scotland

    Tuesday, June 18, 2019 9:07 PM
  • Hi,

    Yes.  I'm reading the error file in a loop and inside that loop, I search for the station file inside a directory and once found, I look for all the formatting errors in specific lines to add a "0" character, using the following.

    My.Computer.FileSystem.OpenTextFileWriter(filePath, False, System.Text.Encoding.ASCII) 

    This is where the error is thrown.

    I don't know how else to go about it.


    Marilyn Gambone

    Wednesday, June 19, 2019 12:10 PM
  • Hi

    Where is the error file being created/written - not in the code you posted. Maybe I am wrong, but shouldn't the Validation code not write it?

    Also, can you show examples of malformed lines that would need to be corrected - is it purely field sizes that need tobe preserved/ amended?


    Regards Les, Livingston, Scotland

    Wednesday, June 19, 2019 1:28 PM
  • Hi,

    You're right.  I've corrected the Validation code to exclude the writing of the error file.  So I have 3 methods:
     - Validate
     - Write To text file
     - Correct the weather data

    If you look at the error sample above, the first line has year = 1900, month = 3, day = 24.  Then in the Weather data, the program looks for that same year, month, day: 

     1900  03  24   0.0007.8100.00004.48

    The number of characters in this line is < 38.  I then need to add "0" to the end of the line so total character count is 38.


    Marilyn Gambone

    Wednesday, June 19, 2019 2:54 PM
  • Hi

    I suspect I am wrong, but, here is code that I *think* does what you describe in your last post. It will cycle through all the files in the path and change the lines to 38 length if less than that, and will write the changes back to original file. Will also write a Log file to the desktop showing changes.

    As I say, I imagine your requirements stretch to more than that, but I can only go on information available.

    The code below is 'short'. It is basically one Sub, one variable and a change ro the calling context to comment out original and add new.

    Option Strict On
    Option Explicit On
    Module Module1
    	Dim stationPath As String = "C:\Users\lesha\Desktop\EpicWeather"
    	' location of epic files
    	Dim errorPath As String = Environment.GetFolderPath(Environment.SpecialFolder.Desktop)
    	' error path
    	Dim errors As ArrayList = New ArrayList()
    	Dim errorFile As String = errorPath & "\" & "EpicFormatError.txt"
    	Sub Main()
    		'	ValidateFormat(stationPath)
    		'	CorrectErrors(errorFile)
    
    		CheckLineLengths()
    	End Sub
    	Dim log As New List(Of String)
    	Private Sub CheckLineLengths()
    		log.Clear()
    		Dim changes As Integer = 0
    		Dim files() As String = IO.Directory.GetFiles(stationPath)
    		For Each file In files
    			Dim lines() As String = IO.File.ReadAllLines(file)
    			Dim amended As Boolean = False
    			For line As Integer = 0 To lines.Count - 1
    				While lines(line).Length < 38
    					log.Add(file)
    					log.Add(lines(line))
    
    					lines(line) = lines(line) & "0"
    					changes += 1
    					log.Add(lines(line))
    					log.Add("-----------")
    
    					amended = True
    				End While
    			Next
    			If amended Then
    				IO.File.WriteAllLines(file, lines)
    				log.Add("Total changes = " & changes.ToString)
    				IO.File.WriteAllLines(errorFile, log.ToArray)
    			End If
    		Next
    	End Sub
    	End Class


    Regards Les, Livingston, Scotland



    • Edited by leshay Wednesday, June 19, 2019 5:17 PM
    • Marked as answer by deskcheck1 Thursday, June 20, 2019 11:48 AM
    Wednesday, June 19, 2019 5:15 PM
  • Hi Les,

    OMG!  I'm blown away!  This has got to be the most elegant solution I've ever seen!

    Thank you! Thank you! Thank you!


    Marilyn Gambone

    Thursday, June 20, 2019 11:49 AM