docs: def addfile(project, filename, user, postdata, inputsource=None,returntype='xml'): #pylint: disable=too-many-return-statements
      """Add a new input file, this invokes the actual uploader"""
  
  
      def errorresponse(msg, code=403):
          if returntype == 'json':
              return withheaders(flask.make_response("{success: false, error: '" + msg + "'}"),'application/json',{'allow_origin': settings.ALLOW_ORIGIN})
          else:
              return withheaders(flask.make_response(msg,403),headers={'allow_origin': settings.ALLOW_ORIGIN}) #(it will have to be explicitly deleted by the client first)
  
      inputtemplate_id = flask.request.headers.get('Inputtemplate','')
      inputtemplate = None
      metadata = None
  
  
      printdebug('Handling addfile, postdata contains fields ' + ",".join(postdata.keys()) )
  
      if 'inputtemplate' in postdata:
          inputtemplate_id = postdata['inputtemplate']
  
      if inputtemplate_id:
          #An input template must always be provided
          for profile in settings.PROFILES:
              for t in profile.input:
                  if t.id == inputtemplate_id:
                      inputtemplate = t
          if not inputtemplate:
              #Inputtemplate not found, send 404
              printlog("Specified inputtemplate (" + inputtemplate_id + ") not found!")
              return withheaders(flask.make_response("Specified inputtemplate (" + inputtemplate_id + ") not found!",404),headers={'allow_origin': settings.ALLOW_ORIGIN})
          printdebug('Inputtemplate explicitly provided: ' + inputtemplate.id )
      if not inputtemplate:
          #See if an inputtemplate is explicitly specified in the filename
          printdebug('Attempting to determine input template from filename ' + filename )
          if '/' in filename.strip('/'):
              raw = filename.split('/')
              inputtemplate = None
              for profile in settings.PROFILES:
                  for it in profile.input:
                      if it.id == raw[0]:
                          inputtemplate = it
                          break
              if inputtemplate:
                  filename = raw[1]
      if not inputtemplate:
          #Check if the specified filename can be uniquely associated with an inputtemplate
          for profile in settings.PROFILES:
              for t in profile.input:
                  if t.filename == filename:
                      if inputtemplate:
                          #we found another one, not unique!! reset and break
                          inputtemplate = None
                          break
                      else:
                          #good, we found one, don't break cause we want to make sure there is only one
                          inputtemplate = t
          if not inputtemplate:
              printlog("No inputtemplate specified and filename " + filename + " does not uniquely match with any inputtemplate!")
              return errorresponse("No inputtemplate specified nor auto-detected for filename " + filename + "!",404)
  
  
  
      #See if other previously uploaded input files use this inputtemplate
      if inputtemplate.unique:
          nextseq = 0 #unique
      else:
          nextseq = 1 #will hold the next sequence number for this inputtemplate (in multi-mode only)
  
      for seq, inputfile in Project.inputindexbytemplate(project, user, inputtemplate): #pylint: disable=unused-variable
          if inputtemplate.unique:
              return errorresponse("You have already submitted a file of this type, you can only submit one. Delete it first. (Inputtemplate=" + inputtemplate.id + ", unique=True)")
          else:
              if seq >= nextseq:
                  nextseq = seq + 1 #next available sequence number
  
  
      if not filename: #Actually, I don't think this can occur at this stage, but we'll leave it in to be sure (yes it can, when the entry shortcut is used!)
          if inputtemplate.filename:
              filename = inputtemplate.filename
          elif inputtemplate.extension:
              filename = str(nextseq) +'-' + str("%034x" % random.getrandbits(128)) + '.' + inputtemplate.extension
          else:
              filename = str(nextseq) +'-' + str("%034x" % random.getrandbits(128))
  
      #Make sure filename matches (only if not an archive)
      if inputtemplate.acceptarchive and (filename[-7:].lower() == '.tar.gz' or filename[-8:].lower() == '.tar.bz2' or filename[-4:].lower() == '.zip'):
          pass
      else:
          if inputtemplate.filename:
              if filename != inputtemplate.filename:
                  filename = inputtemplate.filename
                  #return flask.make_response("Specified filename must the filename dictated by the inputtemplate, which is " + inputtemplate.filename)
              #TODO LATER: add support for calling this with an actual number instead of #
          if inputtemplate.extension:
              if filename[-len(inputtemplate.extension) - 1:].lower() == '.' + inputtemplate.extension.lower():
                  #good, extension matches (case independent). Let's just make sure the case is as defined exactly by the inputtemplate
                  filename = filename[:-len(inputtemplate.extension) - 1] +  '.' + inputtemplate.extension
              else:
                  filename = filename +  '.' + inputtemplate.extension
                  #return flask.make_response("Specified filename does not have the extension dictated by the inputtemplate ("+inputtemplate.extension+")") #403
  
      if inputtemplate.onlyinputsource and (not 'inputsource' in postdata or not postdata['inputsource']):
          return errorresponse("Adding files for this inputtemplate must proceed through inputsource")
  
      if 'converter' in postdata and postdata['converter'] and not postdata['converter'] in [ x.id for x in inputtemplate.converters]:
          return errorresponse("Invalid converter specified: " + postdata['converter'])
  
      #Make sure the filename is secure
      validfilename = True
      DISALLOWED = ('/','&','|','<','>',';','"',"'","`","{","}","\n","\r","\b","\t")
      for c in filename:
          if c in DISALLOWED:
              validfilename = False
              break
  
      if not validfilename:
          return errorresponse("Filename contains invalid symbols! Do not use /,&,|,<,>,',`,\",{,} or ;")
  
  
      #Create the project (no effect if already exists)
      response = Project.create(project, user)
      if response is not None:
          return response
  
  
      printdebug("(Obtaining filename for uploaded file)")
      head = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
      head += "<clamupload>\n"
      if 'file' in flask.request.files:
          printlog("Adding client-side file " + flask.request.files['file'].filename + " to input files")
          sourcefile = flask.request.files['file'].filename
      elif 'url' in postdata and postdata['url']:
          #Download from URL
          printlog("Adding web-based URL " + postdata['url'] + " to input files")
          sourcefile = os.path.basename(postdata['url'])
      elif 'contents' in postdata and postdata['contents']:
          #In message
          printlog("Adding file " + filename + " with explicitly provided contents to input files")
          sourcefile = "editor"
      elif 'inputsource' in postdata and postdata['inputsource']:
          printlog("Adding file " + filename + " from preinstalled data to input files")
          if not inputsource:
              inputsource = None
              for s in inputtemplate.inputsources:
                  if s.id.lower() == postdata['inputsource'].lower():
                      inputsource = s
              if not inputsource:
                  return errorresponse("Specified inputsource '" + postdata['inputsource'] + "' does not exist for inputtemplate '"+inputtemplate.id+"'")
          sourcefile = os.path.basename(inputsource.path)
      elif 'accesstoken' in postdata and 'filename' in postdata:
          #XHR POST, data in body
          printlog("Adding client-side file " + filename + " to input files. Uploaded using XHR POST")
          sourcefile = postdata['filename']
      else:
          return errorresponse("No file, url or contents specified!")
  
  
      #============================ Generate metadata ========================================
      printdebug('(Generating and validating metadata)')
      if 'metafile' in flask.request.files:  #and (not isinstance(postdata['metafile'], dict) or len(postdata['metafile']) > 0)):
          #an explicit metadata file was provided, upload it:
          printlog("Metadata explicitly provided in file, uploading...")
          #Upload file from client to server
          metafile = Project.path(project, user) + 'input/.' + filename + '.METADATA'
          flask.request.files['metafile'].save(metafile)
          try:
              with io.open(metafile,'r',encoding='utf-8') as f:
                  metadata = clam.common.data.CLAMMetaData.fromxml(f.read())
              errors, parameters = inputtemplate.validate(metadata, user)
              validmeta = True
          except Exception as e: #pylint: disable=broad-except
              printlog("Uploaded metadata is invalid! " + str(e))
              metadata = None
              errors = True
              parameters = []
              validmeta = False
      elif 'metadata' in postdata and postdata['metadata']:
          printlog("Metadata explicitly provided in message, uploading...")
          try:
              metadata = clam.common.data.CLAMMetaData.fromxml(postdata['metadata'])
              errors, parameters = inputtemplate.validate(metadata, user)
              validmeta = True
          except: #pylint: disable=bare-except
              printlog("Uploaded metadata is invalid!")
              metadata = None
              errors = True
              parameters = []
              validmeta = False
      elif 'inputsource' in postdata and postdata['inputsource']:
          printlog("Getting metadata from inputsource, uploading...")
          if inputsource.metadata:
              printlog("DEBUG: Validating metadata from inputsource")
              metadata = inputsource.metadata
              errors, parameters = inputtemplate.validate(metadata, user)
              validmeta = True
          else:
              printlog("DEBUG: No metadata provided with inputsource, looking for metadata files..")
              metafilename = os.path.dirname(inputsource.path)
              if metafilename: metafilename += '/'
              metafilename += '.' + os.path.basename(inputsource.path) + '.METADATA'
              if os.path.exists(metafilename):
                  try:
                      metadata = clam.common.data.CLAMMetaData.fromxml(open(metafilename,'r').readlines())
                      errors, parameters = inputtemplate.validate(metadata, user)
                      validmeta = True
                  except: #pylint: disable=bare-except
                      printlog("Uploaded metadata is invalid!")
                      metadata = None
                      errors = True
                      parameters = []
                      validmeta = False
              else:
                  return withheaders(flask.make_response("No metadata found nor specified for inputsource " + inputsource.id ,500),headers={'allow_origin': settings.ALLOW_ORIGIN})
      else:
          errors, parameters = inputtemplate.validate(postdata, user)
          validmeta = True #will be checked later
  
  
      #  ----------- Check if archive are allowed -------------
      archive = False
      addedfiles = []
      if not errors and inputtemplate.acceptarchive: #pylint: disable=too-many-nested-blocks
          printdebug('(Archive test)')
          # -------- Are we an archive? If so, determine what kind
          archivetype = None
          if 'file' in flask.request.files:
              uploadname = sourcefile.lower()
              archivetype = None
              if uploadname[-4:] == '.zip':
                  archivetype = 'zip'
              elif uploadname[-7:] == '.tar.gz':
                  archivetype = 'tar.gz'
              elif uploadname[-4:] == '.tar':
                  archivetype = 'tar'
              elif uploadname[-8:] == '.tar.bz2':
                  archivetype = 'tar.bz2'
              xhrpost = False
          elif 'accesstoken' in postdata and 'filename' in postdata:
              xhrpost = True
              if postdata['filename'][-7:].lower() == '.tar.gz':
                  uploadname = sourcefile.lower()
                  archivetype = 'tar.gz'
              elif postdata['filename'][-8:].lower() == '.tar.bz2':
                  uploadname = sourcefile.lower()
                  archivetype = 'tar.bz2'
              elif postdata['filename'][-4:].lower() == '.tar':
                  uploadname = sourcefile.lower()
                  archivetype = 'tar'
              elif postdata['filename'][-4:].lower() == '.zip':
                  uploadname = sourcefile.lower()
                  archivetype = 'zip'
  
          if archivetype:
              # =============== upload archive ======================
              #random name
              archive = "%032x" % random.getrandbits(128) + '.' + archivetype
  
              #Upload file from client to server
              printdebug('(Archive transfer starting)')
              if not xhrpost:
                  flask.request.files['file'].save(Project.path(project,user) + archive)
              elif xhrpost:
                  with open(Project.path(project,user) + archive,'wb') as f:
                      while True:
                          chunk = flask.request.stream.read(16384)
                          if chunk:
                              f.write(chunk)
                          else:
                              break
              printdebug('(Archive transfer completed)')
              # =============== Extract archive ======================
  
              #Determine extraction command
              if archivetype == 'zip':
                  cmd = 'unzip -u'
              elif archivetype == 'tar':
                  cmd = 'tar -xvf'
              elif archivetype == 'tar.gz':
                  cmd = 'tar -xvzf'
              elif archivetype == 'tar.bz2':
                  cmd = 'tar -xvjf'
              else:
                  raise Exception("Invalid archive format: " + archivetype) #invalid archive, shouldn't happen
  
              #invoke extractor
              printlog("Extracting '" + archive + "'" )
              try:
                  process = subprocess.Popen(cmd + " " + archive, cwd=Project.path(project,user), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
              except: #pylint: disable=bare-except
                  return withheaders(flask.make_response("Unable to extract archive",500),headers={'allow_origin': settings.ALLOW_ORIGIN})
              out, _ = process.communicate() #waits for process to end
  
              if sys.version < '3':
                  if isinstance(out, str):
                      out = unicode(out,'utf-8') #pylint: disable=undefined-variable
              else:
                  if isinstance(out, bytes):
                      out = str(out,'utf-8')
  
              #Read filename results
  
              firstline = True
              for line in out.split("\n"):
                  line = line.strip()
                  if line:
                      printdebug('(Extraction output: ' + line+')')
                      subfile = None
                      if archivetype[0:3] == 'tar':
                          subfile = line
                      elif archivetype == 'zip' and not firstline: #firstline contains archive name itself, skip it
                          colon = line.find(":")
                          if colon:
                              subfile =  line[colon + 1:].strip()
                      if subfile and os.path.isfile(Project.path(project, user) + subfile):
                          subfile_newname = clam.common.data.resolveinputfilename(os.path.basename(subfile), parameters, inputtemplate, nextseq+len(addedfiles), project)
                          printdebug('(Extracted file ' + subfile + ', moving to input/' + subfile_newname+')')
                          os.rename(Project.path(project, user) + subfile, Project.path(project, user) + 'input/' +  subfile_newname)
                          addedfiles.append(subfile_newname)
                  firstline = False
  
              #all done, remove archive
              os.unlink(Project.path(project, user) + archive)
  
      if not archive:
          addedfiles = [clam.common.data.resolveinputfilename(filename, parameters, inputtemplate, nextseq, project)]
  
      fatalerror = None
  
      jsonoutput = {'success': False if errors else True, 'isarchive': archive}
  
  
      output = head
      for filename in addedfiles: #pylint: disable=too-many-nested-blocks
          output += "<upload source=\""+sourcefile +"\" filename=\""+filename+"\" inputtemplate=\"" + inputtemplate.id + "\" templatelabel=\""+inputtemplate.label+"\" format=\""+inputtemplate.formatclass.__name__+"\">\n"
          if not errors:
              output += "<parameters errors=\"no\">"
          else:
              output += "<parameters errors=\"yes\">"
              jsonoutput['error'] = 'There were parameter errors, file not uploaded: '
          for parameter in parameters:
              output += parameter.xml()
              if parameter.error:
                  jsonoutput['error'] += parameter.error + ". "
          output += "</parameters>"
  
  
  
          if not errors:
              if not archive:
                  #============================ Transfer file ========================================
                  printdebug('(Start file transfer: ' +  Project.path(project, user) + 'input/' + filename+' )')
                  if 'file' in flask.request.files:
                      printdebug('(Receiving data by uploading file)')
                      #Upload file from client to server
                      flask.request.files['file'].save(Project.path(project, user) + 'input/' + filename)
                  elif 'url' in postdata and postdata['url']:
                      printdebug('(Receiving data via url)')
                      #Download file from 3rd party server to CLAM server
                      try:
                          r = requests.get(postdata['url'])
                      except:
                          raise flask.abort(404)
                      if not (r.status_code >= 200 and r.status_code < 300):
                          raise flask.abort(404)
  
                      CHUNK = 16 * 1024
                      f = open(Project.path(project, user) + 'input/' + filename,'wb')
                      for chunk in r.iter_content(chunk_size=CHUNK):
                          if chunk: # filter out keep-alive new chunks
                              f.write(chunk)
                              f.flush()
                      f.close()
                  elif 'inputsource' in postdata and postdata['inputsource']:
                      #Copy (symlink!) from preinstalled data
                      printdebug('(Creating symlink to file ' + inputsource.path + ' <- ' + Project.path(project,user) + '/input/ ' + filename + ')')
                      os.symlink(inputsource.path, Project.path(project, user) + 'input/' + filename)
                  elif 'contents' in postdata and postdata['contents']:
                      printdebug('(Receiving data via from contents variable)')
                      #grab encoding
                      encoding = 'utf-8'
                      for p in parameters:
                          if p.id == 'encoding':
                              encoding = p.value
                      #Contents passed in POST message itself
                      try:
                          f = io.open(Project.path(project, user) + 'input/' + filename,'w',encoding=encoding)
                          f.write(postdata['contents'])
                          f.close()
                      except UnicodeError:
                          return errorresponse("Input file " + str(filename) + " is not in the expected encoding!")
                  elif 'accesstoken' in postdata and 'filename' in postdata:
                      printdebug('(Receiving data directly from post body)')
                      with open(Project.path(project,user) + 'input/' + filename,'wb') as f:
                          while True:
                              chunk = flask.request.stream.read(16384)
                              if chunk:
                                  f.write(chunk)
                              else:
                                  break
  
                  printdebug('(File transfer completed)')
  
  
  
              #Create a file object
              file = clam.common.data.CLAMInputFile(Project.path(project, user), filename, False) #get CLAMInputFile without metadata (chicken-egg problem, this does not read the actual file contents!
  
  
  
              #============== Generate metadata ==============
  
              metadataerror = None
              if not metadata and not errors: #check if it has not already been set in another stage
                  printdebug('(Generating metadata)')
                  #for newly generated metadata
                  try:
                      #Now we generate the actual metadata object (unsaved yet though). We pass our earlier validation results to prevent computing it again
                      validmeta, metadata, parameters = inputtemplate.generate(file, (errors, parameters ))
                      if validmeta:
                          #And we tie it to the CLAMFile object
                          file.metadata = metadata
                          #Add inputtemplate ID to metadata
                          metadata.inputtemplate = inputtemplate.id
                      else:
                          metadataerror = "Undefined error"
                  except ValueError as msg:
                      validmeta = False
                      metadataerror = msg
                  except KeyError as msg:
                      validmeta = False
                      metadataerror = msg
              elif validmeta:
                  #for explicitly uploaded metadata
                  metadata.file = file
                  file.metadata = metadata
                  metadata.inputtemplate = inputtemplate.id
  
              if metadataerror:
                  printdebug('(Metadata could not be generated, ' + str(metadataerror) + ',  this usually indicated an error in service configuration)')
                  #output += "<metadataerror />" #This usually indicates an error in service configuration!
                  fatalerror = "<error type=\"metadataerror\">Metadata could not be generated for " + filename + ": " + str(metadataerror) + " (this usually indicates an error in service configuration!)</error>"
                  jsonoutput['error'] = "Metadata could not be generated! " + str(metadataerror) + "  (this usually indicates an error in service configuration!)"
              elif validmeta:
                  #=========== Convert the uploaded file (if requested) ==============
  
                  conversionerror = False
                  if 'converter' in postdata and postdata['converter']:
                      for c in inputtemplate.converters:
                          if c.id == postdata['converter']:
                              converter = c
                              break
                      if converter: #(should always be found, error already provided earlier if not)
                          printdebug('(Invoking converter)')
                          try:
                              success = converter.convertforinput(Project.path(project, user) + 'input/' + filename, metadata)
                          except: #pylint: disable=bare-except
                              success = False
                          if not success:
                              conversionerror = True
                              fatalerror = "<error type=\"conversion\">The file " + xmlescape(filename) + " could not be converted</error>"
                              jsonoutput['error'] = "The file could not be converted"
                              jsonoutput['success'] = False
  
                  #====================== Validate the file itself ====================
                  if not conversionerror:
                      valid = file.validate()
  
                      if valid:
                          printdebug('(Validation ok)')
                          output += "<valid>yes</valid>"
  
                          #Great! Everything ok, save metadata
                          metadata.save(Project.path(project, user) + 'input/' + file.metafilename())
  
                          #And create symbolic link for inputtemplates
                          linkfilename = os.path.dirname(filename)
                          if linkfilename: linkfilename += '/'
                          linkfilename += '.' + os.path.basename(filename) + '.INPUTTEMPLATE' + '.' + inputtemplate.id + '.' + str(nextseq)
                          os.symlink(Project.path(project, user) + 'input/' + filename, Project.path(project, user) + 'input/' + linkfilename)
                      else:
                          printdebug('(Validation error)')
                          #Too bad, everything worked out but the file itself doesn't validate.
                          #output += "<valid>no</valid>"
                          fatalerror = "<error type=\"validation\">The file " + xmlescape(filename) + " did not validate, it is not in the proper expected format.</error>"
                          jsonoutput['error'] = "The file " + filename.replace("'","") + " did not validate, it is not in the proper expected format."
                          jsonoutput['success'] = False
                          #remove upload
                          os.unlink(Project.path(project, user) + 'input/' + filename)
  
  
          output += "</upload>\n"
  
      output += "</clamupload>"
  
  
  
      if returntype == 'boolean':
          return jsonoutput['success']
      elif fatalerror:
          #fatal error return error message with 403 code
          printlog('Fatal Error during upload: ' + fatalerror)
          return errorresponse(head + fatalerror,403)
      elif errors:
          #parameter errors, return XML output with 403 code
          printdebug('There were parameter errors during upload!')
          if returntype == 'json':
              jsonoutput['xml'] = output #embed XML in JSON for complete client-side processing
              return withheaders(flask.make_response(json.dumps(jsonoutput)), 'application/json', {'allow_origin': settings.ALLOW_ORIGIN})
          else:
              return withheaders(flask.make_response(output,403),headers={'allow_origin': settings.ALLOW_ORIGIN})
      elif returntype == 'xml': #success
          printdebug('Returning xml')
          return withheaders(flask.make_response(output), 'text/xml', {'allow_origin': settings.ALLOW_ORIGIN})
      elif returntype == 'json': #success
          printdebug('Returning json')
          #everything ok, return JSON output (caller decides)
          jsonoutput['xml'] = output #embed XML in JSON for complete client-side processing
          return withheaders(flask.make_response(json.dumps(jsonoutput)), 'application/json', {'allow_origin': settings.ALLOW_ORIGIN})
      elif returntype == 'true_on_success':
          return True
      else:
          printdebug('Invalid return type')
          raise Exception("invalid return type")