import pandas as pd
from subprocess import check_output
import sys

fish_df = pd.read_csv(sys.argv[1],names=['fish'])

url_dict = {}
for fish in fish_df.fish:
    output = check_output(['node','scrape_image_urls.js',fish,sys.argv[2]])
    splits = str(output).replace('\\n','').split(' url: ')
    urls = [s.split(',    width')[0][1:-1] for s in splits[1:]]
    url_dict[fish] = urls

url_df = pd.DataFrame(url_dict)
url_df.to_csv(sys.argv[3], sep='|', index=False)
import pandas as pd
from subprocess import check_output
import sys

fish_df = pd.read_csv(sys.argv[1],names=['fish'])

dfs = []
for fish in fish_df.fish:
    output = check_output(['node','scrape_image_urls.js',fish + ' fish',sys.argv[2]])
    splits = str(output).replace('\\n','').split(' url: ')
    urls = [s.split(',    width')[0][1:-1] for s in splits[1:]]
    dfs.append(pd.DataFrame({'fish': fish, 'url': urls}))

out_df = pd.concat(dfs)
out_df.to_csv(sys.argv[3], sep='|', index=False)

Update scrape fish to handle different size url sets and to specify to google that we want fish
Update scrape fish to handle different size url sets and to specify to google that we want fish

  import pandas as pd
  from subprocess import check_output
  import sys
  
  fish_df = pd.read_csv(sys.argv[1],names=['fish'])
  
- url_dict = {}
+ dfs = []
  for fish in fish_df.fish:
-     output = check_output(['node','scrape_image_urls.js',fish,sys.argv[2]])
+     output = check_output(['node','scrape_image_urls.js',fish + ' fish',sys.argv[2]])
      splits = str(output).replace('\\n','').split(' url: ')
      urls = [s.split(',    width')[0][1:-1] for s in splits[1:]]
-     url_dict[fish] = urls
+     dfs.append(pd.DataFrame({'fish': fish, 'url': urls}))
  
- url_df = pd.DataFrame(url_dict)
+ out_df = pd.concat(dfs)
- url_df.to_csv(sys.argv[3], sep='|', index=False)
+ out_df.to_csv(sys.argv[3], sep='|', index=False)
+ 
Update scrape fish to handle different size url sets and to specify to google that we want fish
## Code Before:
import pandas as pd
from subprocess import check_output
import sys

fish_df = pd.read_csv(sys.argv[1],names=['fish'])

url_dict = {}
for fish in fish_df.fish:
    output = check_output(['node','scrape_image_urls.js',fish,sys.argv[2]])
    splits = str(output).replace('\\n','').split(' url: ')
    urls = [s.split(',    width')[0][1:-1] for s in splits[1:]]
    url_dict[fish] = urls

url_df = pd.DataFrame(url_dict)
url_df.to_csv(sys.argv[3], sep='|', index=False)
## Instruction:
Update scrape fish to handle different size url sets and to specify to google that we want fish
## Code After:
import pandas as pd
from subprocess import check_output
import sys

fish_df = pd.read_csv(sys.argv[1],names=['fish'])

dfs = []
for fish in fish_df.fish:
    output = check_output(['node','scrape_image_urls.js',fish + ' fish',sys.argv[2]])
    splits = str(output).replace('\\n','').split(' url: ')
    urls = [s.split(',    width')[0][1:-1] for s in splits[1:]]
    dfs.append(pd.DataFrame({'fish': fish, 'url': urls}))

out_df = pd.concat(dfs)
out_df.to_csv(sys.argv[3], sep='|', index=False)
