(self, cbx_obj, description='',
                           is_ind=False, p_id=None, ind_ci=None)
Parse a CybOX object form a STIX doc. An object can contain
        multiple related_objects, which in turn can have their own
        related_objects, so this handles those recursively.

        :param cbx_obj: The CybOX object to parse.
        :type cbx_obj: A CybOX object.
        :param description: Parent-level (e.g. Observable) description.
        :type description: str
        :param is_ind: Whether the observable is actually an Indicator
        :type is_ind: boolean
        :param p_id: The ID of a parent STIX Indicator or Observable.
        :type p_id: str
        :param ind_ci: The (confidence, impact) of a parent STIX Indicator.
        :type ind_ci: tuple
Parse a CybOX object form a STIX doc. An object can contain
        multiple related_objects, which in turn can have their own
        related_objects, so this handles those recursively.
Parse a CybOX object form a STIX doc . An object can contain multiple related_objects which in turn can have their own related_objects so this handles those recursively .
def parse_cybox_object(self, cbx_obj, description='',
                           is_ind=False, p_id=None, ind_ci=None):
        """
        Parse a CybOX object form a STIX doc. An object can contain
        multiple related_objects, which in turn can have their own
        related_objects, so this handles those recursively.

        :param cbx_obj: The CybOX object to parse.
        :type cbx_obj: A CybOX object.
        :param description: Parent-level (e.g. Observable) description.
        :type description: str
        :param is_ind: Whether the observable is actually an Indicator
        :type is_ind: boolean
        :param p_id: The ID of a parent STIX Indicator or Observable.
        :type p_id: str
        :param ind_ci: The (confidence, impact) of a parent STIX Indicator.
        :type ind_ci: tuple
        """

        # Convert description to unicode if str
        if isinstance(description, str):
            description = description.decode('utf-8')

        # Setup indicator confidence/impact
        if not ind_ci: # if not provided, use defaults
            ind_ci = self.def_ci

        # check for missing attributes
        if not cbx_obj or not cbx_obj.properties:
            if cbx_obj.idref: # just a reference, so nothing to parse
                return
            else:
                cbx_id = getattr(cbx_obj, 'id_', 'None')
                self.failed.append(("No valid object_properties was found!",
                                    "Observable (%s)" % cbx_id,
                                    cbx_id)) # note for display in UI
                return

        # Don't parse if already been parsed
        # This is for artifacts that are related to CybOX File Objects
        if cbx_obj.id_ in self.parsed:
            return

        try: # try to create CRITs object from Cybox Object
            analyst = self.source_instance.analyst
            user = get_user_info(analyst)
            item = cbx_obj.properties
            val = cbx_obj.id_
            if isinstance(item, Address) and not is_ind:
                if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
                                     'ipv4-netmask', 'ipv6-addr',
                                     'ipv6-net', 'ipv6-netmask'):
                    imp_type = "IP"
                    for value in item.address_value.values:
                        val = str(value).strip()
                        if self.preview:
                            res = None
                        else:
                            iptype = get_crits_ip_type(item.category)
                            if iptype:
                                res = ip_add_update(val,
                                                    iptype,
                                                    [self.source],
                                                    analyst=analyst,
                                                    description=description)
                            else:
                                res = {'success': False, 'reason': 'No IP Type'}
                        self.parse_res(imp_type, val, cbx_obj, res, p_id)
            if (not is_ind and (isinstance(item, DomainName) or
                (isinstance(item, URI) and item.type_ == 'Domain Name'))):
                imp_type = "Domain"
                for val in item.value.values:
                    if self.preview:
                        res = None
                    else:
                        res = upsert_domain(str(val),
                                            [self.source],
                                            username=analyst)
                    self.parse_res(imp_type, str(val), cbx_obj, res, p_id)

            elif isinstance(item, HTTPSession):
                imp_type = "RawData"
                dtype = "HTTP Header"
                val = cbx_obj.id_
                try:
                    c_req = item.http_request_response[0].http_client_request
                    hdr = c_req.http_request_header
                    if hdr.raw_header:
                        data = hdr.raw_header.value
                        title = "HTTP Header from STIX: %s" % self.package.id_
                        method = self.source_instance.method
                        ref = self.source_instance.reference
                        if self.preview or not user.has_access_to(RawDataACL.WRITE):
                            res = None
                            val = title
                            rdtype = RawDataType.objects(name=dtype).first()
                            if not rdtype:
                                msg = 'Add Raw Data Type "%s" and try again'
                                res = {'success': False, 'message': msg % dtype}
                        else:
                            res = handle_raw_data_file(data,
                                                    self.source.name,
                                                    user=analyst,
                                                    description=description,
                                                    title=title,
                                                    data_type=dtype,
                                                    tool_name="STIX",
                                                    tool_version=None,
                                                    method=method,
                                                    reference=ref)
                            if not res['success']:
                                if "Invalid data type" in res['message']:
                                    msg = 'Add Raw Data Type "%s" and try again'
                                    res['message'] = msg % dtype
                    else:
                        imp_type = "Indicator"
                        ind_type = IndicatorTypes.USER_AGENT
                        val = hdr.parsed_header.user_agent.value
                        val = ','.join(val) if isinstance(val, list) else val
                        if self.preview or not user.has_access_to(IndicatorACL.WRITE):
                            res = None
                            val = "%s - %s" % (ind_type, val)
                        else:
                            res = handle_indicator_ind(val,
                                                    self.source,
                                                    ind_type,
                                                    IndicatorThreatTypes.UNKNOWN,
                                                    IndicatorAttackTypes.UNKNOWN,
                                                    analyst,
                                                    add_relationship=True,
                                                    description=description,
                                                    confidence=ind_ci[0],
                                                    impact=ind_ci[1])
                except:
                    msg = "Unsupported use of 'HTTPSession' object."
                    res = {'success': False, 'reason': msg}

                self.parse_res(imp_type, val, cbx_obj, res, p_id)
            elif isinstance(item, WhoisEntry):
                # No sure where else to put this
                imp_type = "RawData"
                dtype = "Text"
                val = cbx_obj.id_
                if item.remarks:
                    data = item.remarks.value
                    title = "WHOIS Entry from STIX: %s" % self.package.id_
                    if self.preview or not user.has_access_to(RawDataACL.WRITE):
                        res = None
                        val = title
                        rdtype = RawDataType.objects(name=dtype).first()
                        if not rdtype:
                            msg = 'Add Raw Data Type "%s" and try again'
                            res = {'success': False, 'message': msg % dtype}
                    else:
                        res = handle_raw_data_file(data,
                                                self.source.name,
                                                user=analyst,
                                                description=description,
                                                title=title,
                                                data_type=dtype,
                                                tool_name="WHOIS",
                                                tool_version=None,
                                                method=self.source_instance.method,
                                                reference=self.source_instance.reference)
                        if not res['success']:
                            if "Invalid data type" in res['message']:
                                msg = 'Add Raw Data Type "%s" and try again'
                                res['message'] = msg % dtype
                else:
                    msg = "Unsupported use of 'WhoisEntry' object."
                    res = {'success': False, 'reason': msg}

                self.parse_res(imp_type, val, cbx_obj, res, p_id)
            elif isinstance(item, Artifact):
                # Not sure if this is right, and I believe these can be
                # encoded in a couple different ways.
                imp_type = "RawData"
                dtype = "Text"
                val = cbx_obj.id_
                rawdata = item.data.decode('utf-8')
                # TODO: find out proper ways to determine title, datatype,
                #       tool_name, tool_version
                title = "Artifact for Event: STIX Document %s" % self.package.id_
                if self.preview or not user.has_access_to(RawDataACL.WRITE):
                    res = None
                    val = title
                    rdtype = RawDataType.objects(name=dtype).first()
                    if not rdtype:
                        msg = 'Add Raw Data Type "%s" and try again'
                        res = {'success': False, 'message': msg % dtype}
                else:
                    res = handle_raw_data_file(rawdata,
                                            self.source.name,
                                            user=analyst,
                                            description=description,
                                            title=title,
                                            data_type=dtype,
                                            tool_name="STIX",
                                            tool_version=None,
                                            method=self.source_instance.method,
                                            reference=self.source_instance.reference)
                    if not res['success']:
                        if "Invalid data type" in res['message']:
                            msg = 'Add Raw Data Type "%s" and try again'
                            res['message'] = msg % dtype
                self.parse_res(imp_type, val, cbx_obj, res, p_id)
            elif (isinstance(item, File) and
                  item.custom_properties and
                  item.custom_properties[0].name == "crits_type" and
                  item.custom_properties[0]._value == "Certificate"):
                imp_type = "Certificate"
                val = str(item.file_name)
                data = None
                if self.preview or not user.has_access_to(CertificateACL.WRITE):
                    res = None
                else:
                    for rel_obj in item.parent.related_objects:
                        if isinstance(rel_obj.properties, Artifact):
                            data = rel_obj.properties.data
                            self.parsed.append(rel_obj.id_)
                    res = handle_cert_file(val,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                self.parse_res(imp_type, val, cbx_obj, res, p_id)
            elif isinstance(item, File) and self.has_network_artifact(item):
                imp_type = "PCAP"
                val = str(item.file_name)
                data = None
                if self.preview or not user.has_access_to(PCAPACL.WRITE):
                    res = None
                else:
                    for rel_obj in item.parent.related_objects:
                        if (isinstance(rel_obj.properties, Artifact) and
                            rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
                            data = rel_obj.properties.data
                            self.parsed.append(rel_obj.id_)
                    res = handle_pcap_file(val,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                self.parse_res(imp_type, val, cbx_obj, res, p_id)
            elif isinstance(item, File):
                imp_type = "Sample"
                md5 = item.md5
                if md5:
                    md5 = md5.lower()
                    validate_md5_result = validate_md5_checksum(md5)
                    if validate_md5_result.get('success', False) is False:
                        md5 = None
                sha1 = item.sha1
                if sha1:
                    sha1 = sha1.lower()
                    validate_sha1_result = validate_sha1_checksum(sha1)
                    if validate_sha1_result.get('success', False) is False:
                        sha1 = None
                sha256 = item.sha256
                if sha256:
                    sha256 = sha256.lower()
                    if len(sha256) == 63:
                        sha256 = "0" + sha256
                    validate_sha256_result = validate_sha256_checksum(sha256)
                    if validate_sha256_result.get('success', False) is False:
                        sha256 = None
                ssdeep = getattr(item, 'ssdeep', None) # Not supported yet
                if not ssdeep: # so see if we can find it
                    try:
                        for h in item.hashes.hashes:
                            if h.type_ == 'SSDEEP':
                                ssdeep = str(h.fuzzy_hash_value)
                    except:
                        pass

                fname = None
                if item.file_name is not None:
                    fname = str(item.file_name)
                size = item.size_in_bytes
                data = None
                if item.file_path: # save the path in the description field
                    path = "File Path: " + str(item.file_path)
                    description += "\n" + path
                for rel_obj in item.parent.related_objects or ():
                    if (isinstance(rel_obj.properties, Artifact) and
                        rel_obj.properties.type_ == Artifact.TYPE_FILE):
                        data = rel_obj.properties.data
                        self.parsed.append(rel_obj.id_)
                if not (md5 or data) and (fname or sha1 or sha256 or ssdeep): # Can't create a Sample
                    imp_type = "Indicator"
                    for indt, indv in ((IndicatorTypes.FILE_NAME, fname),
                                       (IndicatorTypes.SHA1, sha1),
                                       (IndicatorTypes.SHA256, sha256),
                                       (IndicatorTypes.SSDEEP, ssdeep)):
                        if indv:
                            if self.preview:
                                res = None
                                indv = "%s - %s" % (indt, indv)
                            else:
                                res = handle_indicator_ind(indv,
                                                  self.source,
                                                  indt,
                                                  IndicatorThreatTypes.UNKNOWN,
                                                  IndicatorAttackTypes.UNKNOWN,
                                                  analyst,
                                                  description=description,
                                                  confidence=ind_ci[0],
                                                  impact=ind_ci[1])
                            self.parse_res(imp_type, indv, cbx_obj, res, p_id)
                elif md5 or data: # Can create a Sample
                    val = fname or md5
                    if self.preview:
                        res = None
                        if fname:
                            val = "%s (%s)" % (md5, fname)
                    elif user.has_access_to(SampleACL.WRITE):
                        res = handle_file(val,
                                          data,
                                          self.source,
                                          user=analyst,
                                          md5_digest=md5,
                                          sha1_digest = sha1,
                                          sha256_digest = sha256,
                                          is_return_only_md5=False,
                                          size=size,
                                          description=description)
                    self.parse_res(imp_type, val, cbx_obj, res, p_id)
                else: # Can't do anything with this object
                    val = cbx_obj.id_
                    msg = "CybOX 'File' object has no hashes, data, or filename"
                    res = {'success': False, 'reason': msg}
                    self.parse_res(imp_type, None, cbx_obj, res, p_id)
            elif isinstance(item, EmailMessage):
                imp_type = 'Email'
                id_list = []
                data = {}
                val = cbx_obj.id_
                get_attach = False
                data['raw_body'] = str(item.raw_body)
                data['raw_header'] = str(item.raw_header)
                data['helo'] = str(item.email_server)
                if item.header:
                    data['subject'] = str(item.header.subject)
                    if item.header.date:
                        data['date'] = item.header.date.value
                    val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
                                                     data['subject'])
                    data['message_id'] = str(item.header.message_id)
                    data['sender'] = str(item.header.sender)
                    data['reply_to'] = str(item.header.reply_to)
                    data['x_originating_ip'] = str(item.header.x_originating_ip)
                    data['x_mailer'] = str(item.header.x_mailer)
                    data['boundary'] = str(item.header.boundary)
                    data['from_address'] = str(item.header.from_)
                    if item.header.to:
                        data['to'] = [str(r) for r in item.header.to.to_list()]

                if data.get('date'): # Email TLOs must have a date
                    data['source'] = self.source.name
                    data['source_method'] = self.source_instance.method
                    data['source_reference'] = self.source_instance.reference
                    if self.preview or user.has_access_to(EmailACL.WRITE):
                        res = None
                    else:
                        res = handle_email_fields(data,
                                                  analyst,
                                                  "STIX")
                    self.parse_res(imp_type, val, cbx_obj, res, p_id)
                    if not self.preview and res.get('status'):
                        id_list.append(cbx_obj.id_) # save ID for atchmnt rels
                        get_attach = True
                else: # Can't be an Email TLO, so save fields
                    for x, key in enumerate(data):
                        if data[key] and data[key] != "None":
                            if key in ('raw_header', 'raw_body'):
                                if key == 'raw_header':
                                    title = "Email Header from STIX Email: %s"
                                    dtype = "Email Header"
                                else:
                                    title = "Email Body from STIX Email: %s"
                                    dtype = "Email Body"
                                imp_type = 'RawData'
                                title = title % cbx_obj.id_
                                if self.preview or not user.has_access_to(RawDataACL.WRITE):
                                    res = None
                                    rdtype = RawDataType.objects(name=dtype)
                                    rdtype = rdtype.first()
                                    if not rdtype:
                                        msg = ('Add Raw Data Type "%s" and try'
                                               ' again')
                                        res = {'success': False,
                                               'message': msg % dtype}
                                else:
                                    res = handle_raw_data_file(data[key],
                                                               self.source,
                                                               analyst,
                                                               description,
                                                               title,
                                                               dtype,
                                                               "STIX",
                                                               self.stix_version)
                                    if not res['success']:
                                        if "Invalid data ty" in res['message']:
                                            msg = ('Add Raw Data Type "%s" and'
                                                   ' try again')
                                            res['message'] = msg % dtype
                                self.parse_res(imp_type, title, cbx_obj,
                                               res, p_id)
                            elif key == 'to':
                                imp_type = 'Target'
                                for y, addr in enumerate(data[key]):
                                    tgt_dict = {'email_address': addr}
                                    if self.preview:
                                        res = None
                                    else:
                                        res = upsert_target(tgt_dict, analyst)
                                        if res['success']:
                                            get_attach = True
                                    tmp_obj = copy(cbx_obj)
                                    tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
                                                                x, y)
                                    self.parse_res(imp_type, addr, tmp_obj,
                                                   res, p_id)
                                    self.idMap.setdefault(cbx_obj.id_,
                                                            []).append(tmp_obj.id_)
                                    id_list.append(tmp_obj.id_)
                            else:
                                imp_type = 'Indicator'
                                if 'reply_to' in key:
                                    ind_type = IndicatorTypes.EMAIL_REPLY_TO
                                elif 'sender' in key:
                                    ind_type = IndicatorTypes.EMAIL_SENDER
                                elif 'from_address' in key:
                                    ind_type = IndicatorTypes.EMAIL_FROM
                                elif 'subject' in key:
                                    ind_type = IndicatorTypes.EMAIL_SUBJECT
                                elif 'x_mailer' in key:
                                    ind_type = IndicatorTypes.EMAIL_X_MAILER
                                elif 'message_id' in key:
                                    ind_type = IndicatorTypes.EMAIL_MESSAGE_ID
                                else:
                                    msg = 'No Indicator type for email field "%s"'
                                    self.failed.append((msg % key,
                                                        "%s (%s)" % (imp_type,
                                                                     data[key]),
                                                        None))
                                    continue
                                if self.preview or not user.has_access_to(IndicatorACL.WRITE):
                                    res = None
                                    data[key] = "%s - %s" % (ind_type,
                                                             data[key])
                                else:
                                    res = handle_indicator_ind(data[key],
                                                          self.source,
                                                          ind_type,
                                                          IndicatorThreatTypes.UNKNOWN,
                                                          IndicatorAttackTypes.UNKNOWN,
                                                          analyst,
                                                          add_domain=True,
                                                          add_relationship=True,
                                                          description=description,
                                                          confidence=ind_ci[0],
                                                          impact=ind_ci[1])
                                    if res['success']:
                                        get_attach = True
                                tmp_obj = copy(cbx_obj)
                                tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
                                self.parse_res(imp_type, data[key], tmp_obj,
                                               res, p_id)
                                self.idMap.setdefault(cbx_obj.id_,
                                                        []).append(tmp_obj.id_)
                                id_list.append(tmp_obj.id_)

                if not self.preview:
                    # Setup relationships between all Email attributes
                    for oid in id_list:
                        for oid2 in id_list:
                            if oid != oid2:
                                self.relationships.append((oid,
                                                           RelationshipTypes.RELATED_TO,
                                                           oid2, "High"))

                    # Should check for attachments and add them here.
                    if get_attach and item.attachments:
                        for attach in item.attachments:
                            rel_id = attach.to_dict()['object_reference']
                            for oid in id_list:
                                self.relationships.append((oid,
                                                           RelationshipTypes.CONTAINS,
                                                           rel_id, "High"))

            else: # try to parse all other possibilities as Indicator
                imp_type = "Indicator"
                val = cbx_obj.id_
                c_obj = make_crits_object(item)

                # Ignore what was already caught above and check for ind_type
                if ((is_ind or c_obj.object_type not in IPTypes.values())
                    and c_obj.object_type):
                    ind_type = c_obj.object_type
                    for val in c_obj.value:
                        if isinstance(val, int):
                            val = unicode(val)
                        elif not val: # skip empty strings
                            continue
                        else:
                            val = val.strip()

                        # handle URIs mislabeled as Domains
                        if (c_obj.object_type == 'Domain'
                            and ('/' in val or ':' in val)):
                            ind_type = "URI"

                        if self.preview or not user.has_access_to(IndicatorACL.WRITE):
                            res = None
                            val = "%s - %s" % (ind_type, val)
                        else:
                            res = handle_indicator_ind(val,
                                                    self.source,
                                                    ind_type,
                                                    IndicatorThreatTypes.UNKNOWN,
                                                    IndicatorAttackTypes.UNKNOWN,
                                                    analyst,
                                                    add_domain=True,
                                                    add_relationship=True,
                                                    description=description,
                                                    confidence=ind_ci[0],
                                                    impact=ind_ci[1])
                        self.parse_res(imp_type, val, cbx_obj, res, p_id)

        except Exception as e: # probably caused by cybox object we don't handle
            self.failed.append((e.message or str(e),
                                "%s (%s)" % (imp_type, val),
                                cbx_obj.id_)) # note for display in UI

        # parse any related CybOX object(s)
        for rel_obj in cbx_obj.related_objects or ():
            self.parse_cybox_object(rel_obj, description, is_ind, p_id, ind_ci)
            self.relationships.append((cbx_obj.id_, rel_obj.relationship.value,
                                       rel_obj.id_ or rel_obj.idref, "High"))
def parse_cybox_object ( self , cbx_obj , description = '' , is_ind = False , p_id = None , ind_ci = None ) : # Convert description to unicode if str if isinstance ( description , str ) : description = description . decode ( 'utf-8' ) # Setup indicator confidence/impact if not ind_ci : # if not provided, use defaults ind_ci = self . def_ci # check for missing attributes if not cbx_obj or not cbx_obj . properties : if cbx_obj . idref : # just a reference, so nothing to parse return else : cbx_id = getattr ( cbx_obj , 'id_' , 'None' ) self . failed . append ( ( "No valid object_properties was found!" , "Observable (%s)" % cbx_id , cbx_id ) ) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj . id_ in self . parsed : return try : # try to create CRITs object from Cybox Object analyst = self . source_instance . analyst user = get_user_info ( analyst ) item = cbx_obj . properties val = cbx_obj . id_ if isinstance ( item , Address ) and not is_ind : if item . category in ( 'cidr' , 'ipv4-addr' , 'ipv4-net' , 'ipv4-netmask' , 'ipv6-addr' , 'ipv6-net' , 'ipv6-netmask' ) : imp_type = "IP" for value in item . address_value . values : val = str ( value ) . strip ( ) if self . preview : res = None else : iptype = get_crits_ip_type ( item . category ) if iptype : res = ip_add_update ( val , iptype , [ self . source ] , analyst = analyst , description = description ) else : res = { 'success' : False , 'reason' : 'No IP Type' } self . parse_res ( imp_type , val , cbx_obj , res , p_id ) if ( not is_ind and ( isinstance ( item , DomainName ) or ( isinstance ( item , URI ) and item . type_ == 'Domain Name' ) ) ) : imp_type = "Domain" for val in item . value . values : if self . preview : res = None else : res = upsert_domain ( str ( val ) , [ self . source ] , username = analyst ) self . parse_res ( imp_type , str ( val ) , cbx_obj , res , p_id ) elif isinstance ( item , HTTPSession ) : imp_type = "RawData" dtype = "HTTP Header" val = cbx_obj . id_ try : c_req = item . http_request_response [ 0 ] . http_client_request hdr = c_req . http_request_header if hdr . raw_header : data = hdr . raw_header . value title = "HTTP Header from STIX: %s" % self . package . id_ method = self . source_instance . method ref = self . source_instance . reference if self . preview or not user . has_access_to ( RawDataACL . WRITE ) : res = None val = title rdtype = RawDataType . objects ( name = dtype ) . first ( ) if not rdtype : msg = 'Add Raw Data Type "%s" and try again' res = { 'success' : False , 'message' : msg % dtype } else : res = handle_raw_data_file ( data , self . source . name , user = analyst , description = description , title = title , data_type = dtype , tool_name = "STIX" , tool_version = None , method = method , reference = ref ) if not res [ 'success' ] : if "Invalid data type" in res [ 'message' ] : msg = 'Add Raw Data Type "%s" and try again' res [ 'message' ] = msg % dtype else : imp_type = "Indicator" ind_type = IndicatorTypes . USER_AGENT val = hdr . parsed_header . user_agent . value val = ',' . join ( val ) if isinstance ( val , list ) else val if self . preview or not user . has_access_to ( IndicatorACL . WRITE ) : res = None val = "%s - %s" % ( ind_type , val ) else : res = handle_indicator_ind ( val , self . source , ind_type , IndicatorThreatTypes . UNKNOWN , IndicatorAttackTypes . UNKNOWN , analyst , add_relationship = True , description = description , confidence = ind_ci [ 0 ] , impact = ind_ci [ 1 ] ) except : msg = "Unsupported use of 'HTTPSession' object." res = { 'success' : False , 'reason' : msg } self . parse_res ( imp_type , val , cbx_obj , res , p_id ) elif isinstance ( item , WhoisEntry ) : # No sure where else to put this imp_type = "RawData" dtype = "Text" val = cbx_obj . id_ if item . remarks : data = item . remarks . value title = "WHOIS Entry from STIX: %s" % self . package . id_ if self . preview or not user . has_access_to ( RawDataACL . WRITE ) : res = None val = title rdtype = RawDataType . objects ( name = dtype ) . first ( ) if not rdtype : msg = 'Add Raw Data Type "%s" and try again' res = { 'success' : False , 'message' : msg % dtype } else : res = handle_raw_data_file ( data , self . source . name , user = analyst , description = description , title = title , data_type = dtype , tool_name = "WHOIS" , tool_version = None , method = self . source_instance . method , reference = self . source_instance . reference ) if not res [ 'success' ] : if "Invalid data type" in res [ 'message' ] : msg = 'Add Raw Data Type "%s" and try again' res [ 'message' ] = msg % dtype else : msg = "Unsupported use of 'WhoisEntry' object." res = { 'success' : False , 'reason' : msg } self . parse_res ( imp_type , val , cbx_obj , res , p_id ) elif isinstance ( item , Artifact ) : # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" dtype = "Text" val = cbx_obj . id_ rawdata = item . data . decode ( 'utf-8' ) # TODO: find out proper ways to determine title, datatype, #       tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self . package . id_ if self . preview or not user . has_access_to ( RawDataACL . WRITE ) : res = None val = title rdtype = RawDataType . objects ( name = dtype ) . first ( ) if not rdtype : msg = 'Add Raw Data Type "%s" and try again' res = { 'success' : False , 'message' : msg % dtype } else : res = handle_raw_data_file ( rawdata , self . source . name , user = analyst , description = description , title = title , data_type = dtype , tool_name = "STIX" , tool_version = None , method = self . source_instance . method , reference = self . source_instance . reference ) if not res [ 'success' ] : if "Invalid data type" in res [ 'message' ] : msg = 'Add Raw Data Type "%s" and try again' res [ 'message' ] = msg % dtype self . parse_res ( imp_type , val , cbx_obj , res , p_id ) elif ( isinstance ( item , File ) and item . custom_properties and item . custom_properties [ 0 ] . name == "crits_type" and item . custom_properties [ 0 ] . _value == "Certificate" ) : imp_type = "Certificate" val = str ( item . file_name ) data = None if self . preview or not user . has_access_to ( CertificateACL . WRITE ) : res = None else : for rel_obj in item . parent . related_objects : if isinstance ( rel_obj . properties , Artifact ) : data = rel_obj . properties . data self . parsed . append ( rel_obj . id_ ) res = handle_cert_file ( val , data , self . source , user = analyst , description = description ) self . parse_res ( imp_type , val , cbx_obj , res , p_id ) elif isinstance ( item , File ) and self . has_network_artifact ( item ) : imp_type = "PCAP" val = str ( item . file_name ) data = None if self . preview or not user . has_access_to ( PCAPACL . WRITE ) : res = None else : for rel_obj in item . parent . related_objects : if ( isinstance ( rel_obj . properties , Artifact ) and rel_obj . properties . type_ == Artifact . TYPE_NETWORK ) : data = rel_obj . properties . data self . parsed . append ( rel_obj . id_ ) res = handle_pcap_file ( val , data , self . source , user = analyst , description = description ) self . parse_res ( imp_type , val , cbx_obj , res , p_id ) elif isinstance ( item , File ) : imp_type = "Sample" md5 = item . md5 if md5 : md5 = md5 . lower ( ) validate_md5_result = validate_md5_checksum ( md5 ) if validate_md5_result . get ( 'success' , False ) is False : md5 = None sha1 = item . sha1 if sha1 : sha1 = sha1 . lower ( ) validate_sha1_result = validate_sha1_checksum ( sha1 ) if validate_sha1_result . get ( 'success' , False ) is False : sha1 = None sha256 = item . sha256 if sha256 : sha256 = sha256 . lower ( ) if len ( sha256 ) == 63 : sha256 = "0" + sha256 validate_sha256_result = validate_sha256_checksum ( sha256 ) if validate_sha256_result . get ( 'success' , False ) is False : sha256 = None ssdeep = getattr ( item , 'ssdeep' , None ) # Not supported yet if not ssdeep : # so see if we can find it try : for h in item . hashes . hashes : if h . type_ == 'SSDEEP' : ssdeep = str ( h . fuzzy_hash_value ) except : pass fname = None if item . file_name is not None : fname = str ( item . file_name ) size = item . size_in_bytes data = None if item . file_path : # save the path in the description field path = "File Path: " + str ( item . file_path ) description += "\n" + path for rel_obj in item . parent . related_objects or ( ) : if ( isinstance ( rel_obj . properties , Artifact ) and rel_obj . properties . type_ == Artifact . TYPE_FILE ) : data = rel_obj . properties . data self . parsed . append ( rel_obj . id_ ) if not ( md5 or data ) and ( fname or sha1 or sha256 or ssdeep ) : # Can't create a Sample imp_type = "Indicator" for indt , indv in ( ( IndicatorTypes . FILE_NAME , fname ) , ( IndicatorTypes . SHA1 , sha1 ) , ( IndicatorTypes . SHA256 , sha256 ) , ( IndicatorTypes . SSDEEP , ssdeep ) ) : if indv : if self . preview : res = None indv = "%s - %s" % ( indt , indv ) else : res = handle_indicator_ind ( indv , self . source , indt , IndicatorThreatTypes . UNKNOWN , IndicatorAttackTypes . UNKNOWN , analyst , description = description , confidence = ind_ci [ 0 ] , impact = ind_ci [ 1 ] ) self . parse_res ( imp_type , indv , cbx_obj , res , p_id ) elif md5 or data : # Can create a Sample val = fname or md5 if self . preview : res = None if fname : val = "%s (%s)" % ( md5 , fname ) elif user . has_access_to ( SampleACL . WRITE ) : res = handle_file ( val , data , self . source , user = analyst , md5_digest = md5 , sha1_digest = sha1 , sha256_digest = sha256 , is_return_only_md5 = False , size = size , description = description ) self . parse_res ( imp_type , val , cbx_obj , res , p_id ) else : # Can't do anything with this object val = cbx_obj . id_ msg = "CybOX 'File' object has no hashes, data, or filename" res = { 'success' : False , 'reason' : msg } self . parse_res ( imp_type , None , cbx_obj , res , p_id ) elif isinstance ( item , EmailMessage ) : imp_type = 'Email' id_list = [ ] data = { } val = cbx_obj . id_ get_attach = False data [ 'raw_body' ] = str ( item . raw_body ) data [ 'raw_header' ] = str ( item . raw_header ) data [ 'helo' ] = str ( item . email_server ) if item . header : data [ 'subject' ] = str ( item . header . subject ) if item . header . date : data [ 'date' ] = item . header . date . value val = "Date: %s, Subject: %s" % ( data . get ( 'date' , 'None' ) , data [ 'subject' ] ) data [ 'message_id' ] = str ( item . header . message_id ) data [ 'sender' ] = str ( item . header . sender ) data [ 'reply_to' ] = str ( item . header . reply_to ) data [ 'x_originating_ip' ] = str ( item . header . x_originating_ip ) data [ 'x_mailer' ] = str ( item . header . x_mailer ) data [ 'boundary' ] = str ( item . header . boundary ) data [ 'from_address' ] = str ( item . header . from_ ) if item . header . to : data [ 'to' ] = [ str ( r ) for r in item . header . to . to_list ( ) ] if data . get ( 'date' ) : # Email TLOs must have a date data [ 'source' ] = self . source . name data [ 'source_method' ] = self . source_instance . method data [ 'source_reference' ] = self . source_instance . reference if self . preview or user . has_access_to ( EmailACL . WRITE ) : res = None else : res = handle_email_fields ( data , analyst , "STIX" ) self . parse_res ( imp_type , val , cbx_obj , res , p_id ) if not self . preview and res . get ( 'status' ) : id_list . append ( cbx_obj . id_ ) # save ID for atchmnt rels get_attach = True else : # Can't be an Email TLO, so save fields for x , key in enumerate ( data ) : if data [ key ] and data [ key ] != "None" : if key in ( 'raw_header' , 'raw_body' ) : if key == 'raw_header' : title = "Email Header from STIX Email: %s" dtype = "Email Header" else : title = "Email Body from STIX Email: %s" dtype = "Email Body" imp_type = 'RawData' title = title % cbx_obj . id_ if self . preview or not user . has_access_to ( RawDataACL . WRITE ) : res = None rdtype = RawDataType . objects ( name = dtype ) rdtype = rdtype . first ( ) if not rdtype : msg = ( 'Add Raw Data Type "%s" and try' ' again' ) res = { 'success' : False , 'message' : msg % dtype } else : res = handle_raw_data_file ( data [ key ] , self . source , analyst , description , title , dtype , "STIX" , self . stix_version ) if not res [ 'success' ] : if "Invalid data ty" in res [ 'message' ] : msg = ( 'Add Raw Data Type "%s" and' ' try again' ) res [ 'message' ] = msg % dtype self . parse_res ( imp_type , title , cbx_obj , res , p_id ) elif key == 'to' : imp_type = 'Target' for y , addr in enumerate ( data [ key ] ) : tgt_dict = { 'email_address' : addr } if self . preview : res = None else : res = upsert_target ( tgt_dict , analyst ) if res [ 'success' ] : get_attach = True tmp_obj = copy ( cbx_obj ) tmp_obj . id_ = '%s-%s-%s' % ( cbx_obj . id_ , x , y ) self . parse_res ( imp_type , addr , tmp_obj , res , p_id ) self . idMap . setdefault ( cbx_obj . id_ , [ ] ) . append ( tmp_obj . id_ ) id_list . append ( tmp_obj . id_ ) else : imp_type = 'Indicator' if 'reply_to' in key : ind_type = IndicatorTypes . EMAIL_REPLY_TO elif 'sender' in key : ind_type = IndicatorTypes . EMAIL_SENDER elif 'from_address' in key : ind_type = IndicatorTypes . EMAIL_FROM elif 'subject' in key : ind_type = IndicatorTypes . EMAIL_SUBJECT elif 'x_mailer' in key : ind_type = IndicatorTypes . EMAIL_X_MAILER elif 'message_id' in key : ind_type = IndicatorTypes . EMAIL_MESSAGE_ID else : msg = 'No Indicator type for email field "%s"' self . failed . append ( ( msg % key , "%s (%s)" % ( imp_type , data [ key ] ) , None ) ) continue if self . preview or not user . has_access_to ( IndicatorACL . WRITE ) : res = None data [ key ] = "%s - %s" % ( ind_type , data [ key ] ) else : res = handle_indicator_ind ( data [ key ] , self . source , ind_type , IndicatorThreatTypes . UNKNOWN , IndicatorAttackTypes . UNKNOWN , analyst , add_domain = True , add_relationship = True , description = description , confidence = ind_ci [ 0 ] , impact = ind_ci [ 1 ] ) if res [ 'success' ] : get_attach = True tmp_obj = copy ( cbx_obj ) tmp_obj . id_ = '%s-%s' % ( cbx_obj . id_ , x ) self . parse_res ( imp_type , data [ key ] , tmp_obj , res , p_id ) self . idMap . setdefault ( cbx_obj . id_ , [ ] ) . append ( tmp_obj . id_ ) id_list . append ( tmp_obj . id_ ) if not self . preview : # Setup relationships between all Email attributes for oid in id_list : for oid2 in id_list : if oid != oid2 : self . relationships . append ( ( oid , RelationshipTypes . RELATED_TO , oid2 , "High" ) ) # Should check for attachments and add them here. if get_attach and item . attachments : for attach in item . attachments : rel_id = attach . to_dict ( ) [ 'object_reference' ] for oid in id_list : self . relationships . append ( ( oid , RelationshipTypes . CONTAINS , rel_id , "High" ) ) else : # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj . id_ c_obj = make_crits_object ( item ) # Ignore what was already caught above and check for ind_type if ( ( is_ind or c_obj . object_type not in IPTypes . values ( ) ) and c_obj . object_type ) : ind_type = c_obj . object_type for val in c_obj . value : if isinstance ( val , int ) : val = unicode ( val ) elif not val : # skip empty strings continue else : val = val . strip ( ) # handle URIs mislabeled as Domains if ( c_obj . object_type == 'Domain' and ( '/' in val or ':' in val ) ) : ind_type = "URI" if self . preview or not user . has_access_to ( IndicatorACL . WRITE ) : res = None val = "%s - %s" % ( ind_type , val ) else : res = handle_indicator_ind ( val , self . source , ind_type , IndicatorThreatTypes . UNKNOWN , IndicatorAttackTypes . UNKNOWN , analyst , add_domain = True , add_relationship = True , description = description , confidence = ind_ci [ 0 ] , impact = ind_ci [ 1 ] ) self . parse_res ( imp_type , val , cbx_obj , res , p_id ) except Exception as e : # probably caused by cybox object we don't handle self . failed . append ( ( e . message or str ( e ) , "%s (%s)" % ( imp_type , val ) , cbx_obj . id_ ) ) # note for display in UI # parse any related CybOX object(s) for rel_obj in cbx_obj . related_objects or ( ) : self . parse_cybox_object ( rel_obj , description , is_ind , p_id , ind_ci ) self . relationships . append ( ( cbx_obj . id_ , rel_obj . relationship . value , rel_obj . id_ or rel_obj . idref , "High" ) )