
import os
from random import choice, randint, random
import string
from subprocess import call
import sys
from tempfile import mkstemp

__author__ = "Uwe L. Korn"
__license__ = "MIT"


input_file = sys.argv[1]
output_file = sys.argv[2]

# Randomly select 1000 lines from the input file and store them temporarily.
temp_f, temp_file = mkstemp()
call(['shuf', '-n', '1000', input_file, '-o', temp_file])

# Modifiy these queries so that they have a non-zero edit distance.
with open(temp_file, 'r') as f:
    with open(output_file, 'w') as out:
        for line in f.readlines():
            if random() > 0.75:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            if random() > 0.25:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            out.write(line)

# Remove the temporary file again.
os.unlink(temp_file)


import codecs
import os
from random import choice, randint, random
import string
from subprocess import call
import sys
from tempfile import mkstemp

__author__ = "Uwe L. Korn"
__license__ = "MIT"


input_file = sys.argv[1]
output_file = sys.argv[2]

# Randomly select 1000 lines from the input file and store them temporarily.
temp_f, temp_file = mkstemp()
call(['shuf', '-n', '1000', input_file, '-o', temp_file])

# Modifiy these queries so that they have a non-zero edit distance.
with codecs.open(temp_file, 'r', 'utf-8') as f:
    with codecs.open(output_file, 'w', 'utf-8') as out:
        for line in f.readlines():
            if random() > 0.75:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            if random() > 0.25:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            out.write(line)

# Remove the temporary file again.
os.unlink(temp_file)

Fix query extractor UTF-8 handling
Fix query extractor UTF-8 handling
  
+ import codecs
  import os
  from random import choice, randint, random
  import string
  from subprocess import call
  import sys
  from tempfile import mkstemp
  
  __author__ = "Uwe L. Korn"
  __license__ = "MIT"
  
  
  input_file = sys.argv[1]
  output_file = sys.argv[2]
  
  # Randomly select 1000 lines from the input file and store them temporarily.
  temp_f, temp_file = mkstemp()
  call(['shuf', '-n', '1000', input_file, '-o', temp_file])
  
  # Modifiy these queries so that they have a non-zero edit distance.
- with open(temp_file, 'r') as f:
+ with codecs.open(temp_file, 'r', 'utf-8') as f:
-     with open(output_file, 'w') as out:
+     with codecs.open(output_file, 'w', 'utf-8') as out:
          for line in f.readlines():
              if random() > 0.75:
                  pos = randint(0, len(line) - 2)
                  line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
              if random() > 0.25:
                  pos = randint(0, len(line) - 2)
                  line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
              out.write(line)
  
  # Remove the temporary file again.
  os.unlink(temp_file)
  
Fix query extractor UTF-8 handling
## Code Before:

import os
from random import choice, randint, random
import string
from subprocess import call
import sys
from tempfile import mkstemp

__author__ = "Uwe L. Korn"
__license__ = "MIT"


input_file = sys.argv[1]
output_file = sys.argv[2]

# Randomly select 1000 lines from the input file and store them temporarily.
temp_f, temp_file = mkstemp()
call(['shuf', '-n', '1000', input_file, '-o', temp_file])

# Modifiy these queries so that they have a non-zero edit distance.
with open(temp_file, 'r') as f:
    with open(output_file, 'w') as out:
        for line in f.readlines():
            if random() > 0.75:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            if random() > 0.25:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            out.write(line)

# Remove the temporary file again.
os.unlink(temp_file)

## Instruction:
Fix query extractor UTF-8 handling
## Code After:

import codecs
import os
from random import choice, randint, random
import string
from subprocess import call
import sys
from tempfile import mkstemp

__author__ = "Uwe L. Korn"
__license__ = "MIT"


input_file = sys.argv[1]
output_file = sys.argv[2]

# Randomly select 1000 lines from the input file and store them temporarily.
temp_f, temp_file = mkstemp()
call(['shuf', '-n', '1000', input_file, '-o', temp_file])

# Modifiy these queries so that they have a non-zero edit distance.
with codecs.open(temp_file, 'r', 'utf-8') as f:
    with codecs.open(output_file, 'w', 'utf-8') as out:
        for line in f.readlines():
            if random() > 0.75:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            if random() > 0.25:
                pos = randint(0, len(line) - 2)
                line = line[0:pos] + choice(string.ascii_lowercase) + line[pos + 1:]
            out.write(line)

# Remove the temporary file again.
os.unlink(temp_file)
