import os
import logging

from aleph.core import db, archive
from aleph.model import Document
from aleph.queues import ingest_entity

log = logging.getLogger(__name__)


def crawl_directory(collection, path, parent=None):
    """Crawl the contents of the given path."""
    content_hash = None
    if not path.is_dir():
        content_hash = archive.archive_file(path)
    foreign_id = path.name
    if parent is not None:
        foreign_id = os.path.join(parent.foreign_id, foreign_id)
    meta = {'file_name': path.name}
    document = Document.save(collection,
                             parent=parent,
                             foreign_id=foreign_id,
                             content_hash=content_hash,
                             meta=meta)
    db.session.commit()
    ingest_entity(collection, document.to_proxy())
    log.info("Crawl [%s]: %s -> %s", collection.id, path, document.id)
    if path.is_dir():
        for child in path.iterdir():
            crawl_directory(collection, child, document)

import os
import logging
from servicelayer.jobs import Job

from aleph.core import db, archive
from aleph.model import Document
from aleph.queues import ingest_entity

log = logging.getLogger(__name__)


def crawl_directory(collection, path, parent=None, job_id=None):
    """Crawl the contents of the given path."""
    content_hash = None
    if not path.is_dir():
        content_hash = archive.archive_file(path)
    foreign_id = path.name
    if parent is not None:
        foreign_id = os.path.join(parent.foreign_id, foreign_id)
    meta = {'file_name': path.name}
    document = Document.save(collection,
                             parent=parent,
                             foreign_id=foreign_id,
                             content_hash=content_hash,
                             meta=meta)
    db.session.commit()
    job_id = job_id or Job.random_id()
    ingest_entity(collection, document.to_proxy(), job_id=job_id)
    log.info("Crawl [%s]: %s -> %s", collection.id, path, document.id)
    if path.is_dir():
        for child in path.iterdir():
            crawl_directory(collection, child, document, job_id)

Make stable job IDs in ingest runs
Make stable job IDs in ingest runs

  import os
  import logging
+ from servicelayer.jobs import Job
  
  from aleph.core import db, archive
  from aleph.model import Document
  from aleph.queues import ingest_entity
  
  log = logging.getLogger(__name__)
  
  
- def crawl_directory(collection, path, parent=None):
+ def crawl_directory(collection, path, parent=None, job_id=None):
      """Crawl the contents of the given path."""
      content_hash = None
      if not path.is_dir():
          content_hash = archive.archive_file(path)
      foreign_id = path.name
      if parent is not None:
          foreign_id = os.path.join(parent.foreign_id, foreign_id)
      meta = {'file_name': path.name}
      document = Document.save(collection,
                               parent=parent,
                               foreign_id=foreign_id,
                               content_hash=content_hash,
                               meta=meta)
      db.session.commit()
+     job_id = job_id or Job.random_id()
-     ingest_entity(collection, document.to_proxy())
+     ingest_entity(collection, document.to_proxy(), job_id=job_id)
      log.info("Crawl [%s]: %s -> %s", collection.id, path, document.id)
      if path.is_dir():
          for child in path.iterdir():
-             crawl_directory(collection, child, document)
+             crawl_directory(collection, child, document, job_id)
  
Make stable job IDs in ingest runs
## Code Before:
import os
import logging

from aleph.core import db, archive
from aleph.model import Document
from aleph.queues import ingest_entity

log = logging.getLogger(__name__)


def crawl_directory(collection, path, parent=None):
    """Crawl the contents of the given path."""
    content_hash = None
    if not path.is_dir():
        content_hash = archive.archive_file(path)
    foreign_id = path.name
    if parent is not None:
        foreign_id = os.path.join(parent.foreign_id, foreign_id)
    meta = {'file_name': path.name}
    document = Document.save(collection,
                             parent=parent,
                             foreign_id=foreign_id,
                             content_hash=content_hash,
                             meta=meta)
    db.session.commit()
    ingest_entity(collection, document.to_proxy())
    log.info("Crawl [%s]: %s -> %s", collection.id, path, document.id)
    if path.is_dir():
        for child in path.iterdir():
            crawl_directory(collection, child, document)

## Instruction:
Make stable job IDs in ingest runs
## Code After:
import os
import logging
from servicelayer.jobs import Job

from aleph.core import db, archive
from aleph.model import Document
from aleph.queues import ingest_entity

log = logging.getLogger(__name__)


def crawl_directory(collection, path, parent=None, job_id=None):
    """Crawl the contents of the given path."""
    content_hash = None
    if not path.is_dir():
        content_hash = archive.archive_file(path)
    foreign_id = path.name
    if parent is not None:
        foreign_id = os.path.join(parent.foreign_id, foreign_id)
    meta = {'file_name': path.name}
    document = Document.save(collection,
                             parent=parent,
                             foreign_id=foreign_id,
                             content_hash=content_hash,
                             meta=meta)
    db.session.commit()
    job_id = job_id or Job.random_id()
    ingest_entity(collection, document.to_proxy(), job_id=job_id)
    log.info("Crawl [%s]: %s -> %s", collection.id, path, document.id)
    if path.is_dir():
        for child in path.iterdir():
            crawl_directory(collection, child, document, job_id)
