diff --git a/data/SPDXRdfExample.rdf b/data/SPDXRdfExample.rdf index 0f8f21a4d..fc50db092 100644 --- a/data/SPDXRdfExample.rdf +++ b/data/SPDXRdfExample.rdf @@ -3,6 +3,17 @@ xmlns:j.0="http://usefulinc.com/ns/doap#" xmlns="http://spdx.org/rdf/terms#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"> + + + + + from linux kernel + Copyright 2008-2010 John Smith + The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz. + This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later. + + + Sample_Document-V2.1 diff --git a/data/SPDXTagExample.tag b/data/SPDXTagExample.tag index 7ed3bb927..01b3e150f 100644 --- a/data/SPDXTagExample.tag +++ b/data/SPDXTagExample.tag @@ -77,6 +77,15 @@ ArtifactOfProjectHomePage: http://www.openjena.org/ ArtifactOfProjectURI: UNKNOWN FileComment: This file belongs to Jena +## Snippet Information +SnippetSPDXID: SPDXRef-Snippet +SnippetFromFileSPDXID: SPDXRef-DoapSource +SnippetLicenseComments: The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz. +SnippetCopyrightText: Copyright 2008-2010 John Smith +SnippetComment: This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later. +SnippetName: from linux kernel +SnippetLicenseConcluded: Apache-2.0 +LicenseInfoInSnippet: Apache-2.0 ## License Information LicenseID: LicenseRef-3 diff --git a/spdx/document.py b/spdx/document.py index 4c8cecc4a..62708999f 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -266,6 +266,7 @@ class Document(object): Type: Review. - annotations: SPDX document annotation information, Optional zero or more. Type: Annotation. + - snippet: Snippet information. Optional zero or more. Type: Snippet. """ def __init__(self, version=None, data_license=None, name=None, spdx_id=None, @@ -284,6 +285,7 @@ def __init__(self, version=None, data_license=None, name=None, spdx_id=None, self.extracted_licenses = [] self.reviews = [] self.annotations = [] + self.snippet = [] def add_review(self, review): self.reviews.append(review) @@ -297,6 +299,9 @@ def add_extr_lic(self, lic): def add_ext_document_reference(self, ext_doc_ref): self.ext_document_references.append(ext_doc_ref) + def add_snippet(self, snip): + self.snippet.append(snip) + @property def files(self): return self.package.files @@ -324,6 +329,7 @@ def validate(self, messages): messages = self.validate_package(messages) messages = self.validate_extracted_licenses(messages) messages = self.validate_reviews(messages) + messages = self.validate_snippet(messages) return messages @@ -389,6 +395,12 @@ def validate_annotations(self, messages): return messages + def validate_snippet(self, messages=None): + for snippet in self.snippet: + messages = snippet.validate(messages) + + return messages + def validate_creation_info(self, messages): if self.creation_info is not None: messages = self.creation_info.validate(messages) diff --git a/spdx/parsers/lexers/tagvalue.py b/spdx/parsers/lexers/tagvalue.py index 70f7de60c..25db8ac1c 100644 --- a/spdx/parsers/lexers/tagvalue.py +++ b/spdx/parsers/lexers/tagvalue.py @@ -80,6 +80,15 @@ class Lexer(object): 'LicenseName': 'LICS_NAME', 'LicenseCrossReference': 'LICS_CRS_REF', 'LicenseComment': 'LICS_COMMENT', + # Snippet + 'SnippetSPDXID': 'SNIPPET_SPDX_ID', + 'SnippetName': 'SNIPPET_NAME', + 'SnippetComment': 'SNIPPET_COMMENT', + 'SnippetCopyrightText': 'SNIPPET_CR_TEXT', + 'SnippetLicenseComments': 'SNIPPET_LICS_COMMENT', + 'SnippetFromFileSPDXID': 'SNIPPET_FILE_SPDXID', + 'SnippetLicenseConcluded': 'SNIPPET_LICS_CONC', + 'LicenseInfoInSnippet': 'SNIPPET_LICS_INFO', # Common 'NOASSERTION': 'NO_ASSERT', 'UNKNOWN': 'UN_KNOWN', diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index ef196b22f..7aa98a501 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -55,7 +55,12 @@ 'REVIEWER_VALUE' : 'Invalid reviewer value \'{0}\' must be Organization, Tool or Person.', 'REVIEW_DATE' : 'Invalid review date value \'{0}\' must be date in ISO 8601 format.', 'ANNOTATOR_VALUE': 'Invalid annotator value \'{0}\' must be Organization, Tool or Person.', - 'ANNOTATION_DATE': 'Invalid annotation date value \'{0}\' must be date in ISO 8601 format.' + 'ANNOTATION_DATE': 'Invalid annotation date value \'{0}\' must be date in ISO 8601 format.', + 'SNIPPET_SPDX_ID_VALUE' : 'SPDXID must be "SPDXRef-[idstring]" where [idstring] is a unique string ' + 'containing letters, numbers, ".", "-".', + 'SNIPPET_SINGLE_LICS' : 'Snippet Concluded License must be a license url or spdx:noassertion or spdx:none.', + 'SNIPPET_LIC_INFO' : 'License Information in Snippet must be a license url or a reference ' + 'to the license, denoted by LicenseRef-[idstring] or spdx:noassertion or spdx:none.', } @@ -677,6 +682,87 @@ def p_file_lic_conc(self, f_term, predicate): self.more_than_one_error('file {0}'.format(predicate)) +class SnippetParser(LicenseParser): + """ + Helper class for parsing snippet information. + """ + + def __init__(self, builder, logger): + super(SnippetParser, self).__init__(builder, logger) + + def parse_snippet(self, snippet_term): + try: + self.builder.create_snippet(self.doc, snippet_term) + except SPDXValueError: + self.value_error('SNIPPET_SPDX_ID_VALUE', snippet_term) + + for _s, _p, o in self.graph.triples((snippet_term, self.spdx_namespace['name'], None)): + try: + self.builder.set_snippet_name(self.doc, six.text_type(o)) + except CardinalityError: + self.more_than_one_error('snippetName') + break + + for _s, _p, o in self.graph.triples((snippet_term, self.spdx_namespace['licenseComments'], None)): + try: + self.builder.set_snippet_lic_comment(self.doc, six.text_type(o)) + except CardinalityError: + self.more_than_one_error('licenseComments') + break + + for _s, _p, o in self.graph.triples((snippet_term, RDFS.comment, None)): + try: + self.builder.set_snippet_comment(self.doc, six.text_type(o)) + except CardinalityError: + self.more_than_one_error('comment') + break + + for _s, _p, o in self.graph.triples((snippet_term, self.spdx_namespace['copyrightText'], None)): + try: + self.builder.set_snippet_copyright(self.doc, self.to_special_value(six.text_type(o))) + except CardinalityError: + self.more_than_one_error('copyrightText') + break + + try: + for _, _, licenses in self.graph.triples( + (snippet_term, self.spdx_namespace['licenseConcluded'], None)): + if (licenses, RDF.type, self.spdx_namespace['ConjunctiveLicenseSet']) in self.graph: + lics = self.handle_conjunctive_list(licenses) + self.builder.set_snip_concluded_license(self.doc, lics) + + elif (licenses, RDF.type, self.spdx_namespace['DisjunctiveLicenseSet']) in self.graph: + lics = self.handle_disjunctive_list(licenses) + self.builder.set_snip_concluded_license(self.doc, lics) + + else: + try: + lics = self.handle_lics(licenses) + self.builder.set_snip_concluded_license(self.doc, lics) + except SPDXValueError: + self.value_error('SNIPPET_SINGLE_LICS', licenses) + except CardinalityError: + self.more_than_one_error('package {0}'.format( + self.spdx_namespace['licenseConcluded'])) + + for _, _, info in self.graph.triples( + (snippet_term, self.spdx_namespace['licenseInfoInSnippet'], None)): + lic = self.handle_lics(info) + if lic is not None: + try: + self.builder.set_snippet_lics_info(self.doc, lic) + except SPDXValueError: + self.value_error('SNIPPET_LIC_INFO', lic) + + for _s, _p, o in self.graph.triples( + (snippet_term, self.spdx_namespace['snippetFromFile'], None)): + try: + self.builder.set_snip_from_file_spdxid(self.doc, six.text_type(o)) + except CardinalityError: + self.more_than_one_error('snippetFromFile') + break + + class ReviewParser(BaseParser): """ Helper class for parsing review information. @@ -824,7 +910,7 @@ def get_annotator(self, r_term): self.value_error('ANNOTATOR_VALUE', annotator_list[0][2]) -class Parser(PackageParser, FileParser, ReviewParser, AnnotationParser): +class Parser(PackageParser, FileParser, SnippetParser, ReviewParser, AnnotationParser): """ RDF/XML file parser. """ @@ -856,6 +942,9 @@ def parse(self, fil): for s, _p, o in self.graph.triples((None, self.spdx_namespace['referencesFile'], None)): self.parse_file(o) + for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['Snippet'])): + self.parse_snippet(s) + for s, _p, o in self.graph.triples((None, self.spdx_namespace['reviewed'], None)): self.parse_review(o) diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 71921453c..7e2eec5c7 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -364,6 +364,50 @@ def set_file_notice(self, doc, text): raise OrderError('File::Notice') +class SnippetBuilder(tagvaluebuilders.SnippetBuilder): + + def __init__(self): + super(SnippetBuilder, self).__init__() + + def set_snippet_lic_comment(self, doc, lic_comment): + """Sets the snippet's license comment. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if already set. + """ + self.assert_snippet_exists() + if not self.snippet_lic_comment_set: + self.snippet_lic_comment_set = True + doc.snippet[-1].license_comment = lic_comment + else: + CardinalityError('Snippet::licenseComments') + + def set_snippet_comment(self, doc, comment): + """ + Sets general comments about the snippet. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if comment already set. + """ + self.assert_snippet_exists() + if not self.snippet_comment_set: + self.snippet_comment_set = True + doc.snippet[-1].comment = comment + return True + else: + raise CardinalityError('Snippet::comment') + + def set_snippet_copyright(self, doc, copyright): + """Sets the snippet's copyright text. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if already set. + """ + self.assert_snippet_exists() + if not self.snippet_copyright_set: + self.snippet_copyright_set = True + doc.snippet[-1].copyright = copyright + else: + raise CardinalityError('Snippet::copyrightText') + + class ReviewBuilder(tagvaluebuilders.ReviewBuilder): def __init__(self): @@ -426,7 +470,7 @@ def add_annotation_type(self, doc, annotation_type): class Builder(DocBuilder, EntityBuilder, CreationInfoBuilder, PackageBuilder, - FileBuilder, ReviewBuilder, ExternalDocumentRefBuilder, + FileBuilder, SnippetBuilder, ReviewBuilder, ExternalDocumentRefBuilder, AnnotationBuilder): def __init__(self): diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index 412212ec9..8be3efe58 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -99,6 +99,19 @@ 'LICS_COMMENT_VALUE' : 'LicenseComment must be free form text, line: {0}', 'LICS_CRS_REF_VALUE' : 'LicenseCrossReference must be uri as single line of text, line: {0}', 'PKG_CPY_TEXT_VALUE' : 'Package copyright text must be free form text, line: {0}', + 'SNIP_SPDX_ID_VALUE' : 'SPDXID must be "SPDXRef-[idstring]" where [idstring] is a unique string ' + 'containing letters, numbers, ".", "-".', + 'SNIPPET_NAME_VALUE' : 'SnippetName must be a single line of text, line: {0}', + 'SNIP_COMMENT_VALUE' : 'SnippetComment must be free form text, line: {0}', + 'SNIP_COPYRIGHT_VALUE' : 'SnippetCopyrightText must be one of NOASSERTION, NONE or free form text, line: {0}', + 'SNIP_LICS_COMMENT_VALUE' : 'SnippetLicenseComments must be free form text, line: {0}', + 'SNIP_FILE_SPDXID_VALUE' : 'SnippetFromFileSPDXID must be ["DocumentRef-"[idstring]":"] SPDXID ' + 'where DocumentRef-[idstring]: is an optional reference to an external' + 'SPDX Document and SPDXID is a string containing letters, ' + 'numbers, ".", "-".', + 'SNIP_LICS_CONC_VALUE': 'SnippetLicenseConcluded must be NOASSERTION, NONE, license identifier ' + 'or license list, line:{0}', + 'SNIP_LICS_INFO_VALUE': 'LicenseInfoInSnippet must be NOASSERTION, NONE or license identifier, line: {0}', } @@ -169,6 +182,14 @@ def p_attrib(self, p): | file_contrib | file_dep | file_artifact + | snip_spdx_id + | snip_name + | snip_comment + | snip_cr_text + | snip_lic_comment + | snip_file_spdx_id + | snip_lics_conc + | snip_lics_info | extr_lic_id | extr_lic_text | extr_lic_name @@ -1027,6 +1048,201 @@ def p_package_name_1(self, p): msg = ERROR_MESSAGES['PACKAGE_NAME_VALUE'].format(p.lineno(1)) self.logger.log(msg) + def p_snip_spdx_id(self, p): + """snip_spdx_id : SNIPPET_SPDX_ID LINE""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.create_snippet(self.document, value) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_SPDX_ID_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + + def p_snip_spdx_id_1(self, p): + """snip_spdx_id : SNIPPET_SPDX_ID error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_SPDX_ID_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_name(self, p): + """snip_name : SNIPPET_NAME LINE""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_snippet_name(self.document, value) + except OrderError: + self.order_error('SnippetName', 'SnippetSPDXID', p.lineno(1)) + except CardinalityError: + self.more_than_one_error('SnippetName', p.lineno(1)) + + def p_snippet_name_1(self, p): + """snip_name : SNIPPET_NAME error""" + self.error = True + msg = ERROR_MESSAGES['SNIPPET_NAME_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_comment(self, p): + """snip_comment : SNIPPET_COMMENT TEXT""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_snippet_comment(self.document, value) + except OrderError: + self.order_error('SnippetComment', 'SnippetSPDXID', p.lineno(1)) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_COMMENT_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + except CardinalityError: + self.more_than_one_error('SnippetComment', p.lineno(1)) + + def p_snippet_comment_1(self, p): + """snip_comment : SNIPPET_COMMENT error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_COMMENT_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_cr_text(self, p): + """snip_cr_text : SNIPPET_CR_TEXT snip_cr_value""" + try: + self.builder.set_snippet_copyright(self.document, p[2]) + except OrderError: + self.order_error('SnippetCopyrightText', 'SnippetSPDXID', p.lineno(1)) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_COPYRIGHT_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + except CardinalityError: + self.more_than_one_error('SnippetCopyrightText', p.lineno(1)) + + def p_snippet_cr_text_1(self, p): + """snip_cr_text : SNIPPET_CR_TEXT error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_COPYRIGHT_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_cr_value_1(self, p): + """snip_cr_value : TEXT""" + if six.PY2: + p[0] = p[1].decode(encoding='utf-8') + else: + p[0] = p[1] + + def p_snippet_cr_value_2(self, p): + """snip_cr_value : NONE""" + p[0] = utils.SPDXNone() + + def p_snippet_cr_value_3(self, p): + """snip_cr_value : NO_ASSERT""" + p[0] = utils.NoAssert() + + def p_snippet_lic_comment(self, p): + """snip_lic_comment : SNIPPET_LICS_COMMENT TEXT""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_snippet_lic_comment(self.document, value) + except OrderError: + self.order_error('SnippetLicenseComments', 'SnippetSPDXID', p.lineno(1)) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_LICS_COMMENT_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + except CardinalityError: + self.more_than_one_error('SnippetLicenseComments', p.lineno(1)) + + def p_snippet_lic_comment_1(self, p): + """snip_lic_comment : SNIPPET_LICS_COMMENT error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_LICS_COMMENT_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snip_from_file_spdxid(self, p): + """snip_file_spdx_id : SNIPPET_FILE_SPDXID LINE""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_snip_from_file_spdxid(self.document, value) + except OrderError: + self.order_error('SnippetFromFileSPDXID', 'SnippetSPDXID', p.lineno(1)) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_FILE_SPDXID_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + except CardinalityError: + self.more_than_one_error('SnippetFromFileSPDXID', p.lineno(1)) + + def p_snip_from_file_spdxid_1(self, p): + """snip_file_spdx_id : SNIPPET_FILE_SPDXID error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_FILE_SPDXID_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_concluded_license(self, p): + """snip_lics_conc : SNIPPET_LICS_CONC conc_license""" + try: + self.builder.set_snip_concluded_license(self.document, p[2]) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_LICS_CONC_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + except OrderError: + self.order_error('SnippetLicenseConcluded', + 'SnippetSPDXID', p.lineno(1)) + except CardinalityError: + self.more_than_one_error('SnippetLicenseConcluded', p.lineno(1)) + + def p_snippet_concluded_license_1(self, p): + """snip_lics_conc : SNIPPET_LICS_CONC error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_LICS_CONC_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_lics_info(self, p): + """snip_lics_info : SNIPPET_LICS_INFO snip_lic_info_value""" + try: + self.builder.set_snippet_lics_info(self.document, p[2]) + except OrderError: + self.order_error( + 'LicenseInfoInSnippet', 'SnippetSPDXID', p.lineno(1)) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['SNIP_LICS_INFO_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snippet_lics_info_1(self, p): + """snip_lics_info : SNIPPET_LICS_INFO error""" + self.error = True + msg = ERROR_MESSAGES['SNIP_LICS_INFO_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_snip_lic_info_value_1(self, p): + """snip_lic_info_value : NONE""" + p[0] = utils.SPDXNone() + + def p_snip_lic_info_value_2(self, p): + """snip_lic_info_value : NO_ASSERT""" + p[0] = utils.NoAssert() + + def p_snip_lic_info_value_3(self, p): + """snip_lic_info_value : LINE""" + if six.PY2: + value = p[1].decode(encoding='utf-8') + else: + value = p[1] + p[0] = document.License.from_identifier(value) + def p_reviewer_1(self, p): """reviewer : REVIEWER entity""" self.builder.add_reviewer(self.document, p[2]) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 6213f34c3..2334e00ab 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -25,6 +25,7 @@ from spdx import file from spdx import package from spdx import review +from spdx import snippet from spdx import utils from spdx import version @@ -1142,8 +1143,159 @@ def reset_extr_lics(self): self.extr_lic_comment_set = False +class SnippetBuilder(object): + + def __init__(self): + # FIXME: this state does not make sense + self.reset_snippet() + + def create_snippet(self, doc, spdx_id): + """Creates a snippet for the SPDX Document. + spdx_id - To uniquely identify any element in an SPDX document which + may be referenced by other elements. + Raises SPDXValueError if the data is a malformed value. + """ + self.reset_snippet() + spdx_id = spdx_id.split('#')[-1] + if validations.validate_snippet_spdx_id(spdx_id): + doc.add_snippet(snippet.Snippet(spdx_id=spdx_id)) + self.snippet_spdx_id_set = True + return True + else: + raise SPDXValueError('Snippet::SnippetSPDXID') + + def set_snippet_name(self, doc, name): + """ + Sets name of the snippet. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if the name is already set. + """ + self.assert_snippet_exists() + if not self.snippet_name_set: + self.snippet_name_set = True + doc.snippet[-1].name = name + return True + else: + raise CardinalityError('SnippetName') + + def set_snippet_comment(self, doc, comment): + """ + Sets general comments about the snippet. + Raises OrderError if no snippet previously defined. + Raises SPDXValueError if the data is a malformed value. + Raises CardinalityError if comment already set. + """ + self.assert_snippet_exists() + if not self.snippet_comment_set: + self.snippet_comment_set = True + if validations.validate_snip_comment(comment): + doc.snippet[-1].comment = str_from_text(comment) + return True + else: + raise SPDXValueError('Snippet::SnippetComment') + else: + raise CardinalityError('Snippet::SnippetComment') + + def set_snippet_copyright(self, doc, text): + """Sets the snippet's copyright text. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if already set. + Raises SPDXValueError if text is not one of [None, NOASSERT, TEXT]. + """ + self.assert_snippet_exists() + if not self.snippet_copyright_set: + self.snippet_copyright_set = True + if validations.validate_snippet_copyright(text): + if isinstance(text, string_types): + doc.snippet[-1].copyright = str_from_text(text) + else: + doc.snippet[-1].copyright = text # None or NoAssert + else: + raise SPDXValueError('Snippet::SnippetCopyrightText') + else: + raise CardinalityError('Snippet::SnippetCopyrightText') + + def set_snippet_lic_comment(self, doc, text): + """Sets the snippet's license comment. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if already set. + Raises SPDXValueError if the data is a malformed value. + """ + self.assert_snippet_exists() + if not self.snippet_lic_comment_set: + self.snippet_lic_comment_set = True + if validations.validate_snip_lic_comment(text): + doc.snippet[-1].license_comment = str_from_text(text) + return True + else: + raise SPDXValueError('Snippet::SnippetLicenseComments') + else: + raise CardinalityError('Snippet::SnippetLicenseComments') + + def set_snip_from_file_spdxid(self, doc, snip_from_file_spdxid): + """Sets the snippet's 'Snippet from File SPDX Identifier'. + Raises OrderError if no snippet previously defined. + Raises CardinalityError if already set. + Raises SPDXValueError if the data is a malformed value. + """ + self.assert_snippet_exists() + snip_from_file_spdxid = snip_from_file_spdxid.split('#')[-1] + if not self.snip_file_spdxid_set: + self.snip_file_spdxid_set = True + if validations.validate_snip_file_spdxid(snip_from_file_spdxid): + doc.snippet[-1].snip_from_file_spdxid = snip_from_file_spdxid + return True + else: + raise SPDXValueError('Snippet::SnippetFromFileSPDXID') + else: + raise CardinalityError('Snippet::SnippetFromFileSPDXID') + + def set_snip_concluded_license(self, doc, conc_lics): + """ + Raises OrderError if no snippet previously defined. + Raises CardinalityError if already set. + Raises SPDXValueError if the data is a malformed value. + """ + self.assert_snippet_exists() + if not self.snippet_conc_lics_set: + self.snippet_conc_lics_set = True + if validations.validate_lics_conc(conc_lics): + doc.snippet[-1].conc_lics = conc_lics + return True + else: + raise SPDXValueError('Snippet::SnippetLicenseConcluded') + else: + raise CardinalityError('Snippet::SnippetLicenseConcluded') + + def set_snippet_lics_info(self, doc, lics_info): + """ + Raises OrderError if no snippet previously defined. + Raises SPDXValueError if the data is a malformed value. + """ + self.assert_snippet_exists() + if validations.validate_snip_lics_info(lics_info): + doc.snippet[-1].add_lics(lics_info) + return True + else: + raise SPDXValueError('Snippet::LicenseInfoInSnippet') + + def reset_snippet(self): + # FIXME: this state does not make sense + self.snippet_spdx_id_set = False + self.snippet_name_set = False + self.snippet_comment_set = False + self.snippet_copyright_set = False + self.snippet_lic_comment_set = False + self.snip_file_spdxid_set = False + self.snippet_conc_lics_set = False + + def assert_snippet_exists(self): + if not self.snippet_spdx_id_set: + raise OrderError('Snippet') + + class Builder(DocBuilder, CreationInfoBuilder, EntityBuilder, ReviewBuilder, - PackageBuilder, FileBuilder, LicenseBuilder, + PackageBuilder, FileBuilder, LicenseBuilder, SnippetBuilder, ExternalDocumentRefBuilder, AnnotationBuilder): """SPDX document builder.""" @@ -1165,3 +1317,4 @@ def reset(self): self.reset_reviews() self.reset_annotations() self.reset_extr_lics() + self.reset_snippet() diff --git a/spdx/parsers/validations.py b/spdx/parsers/validations.py index 24ff5055c..e3f9727a7 100644 --- a/spdx/parsers/validations.py +++ b/spdx/parsers/validations.py @@ -235,3 +235,47 @@ def validate_extr_lic_name(value, optional=False): return optional else: return isinstance(value, (six.string_types, utils.NoAssert, rdflib.Literal)) + + +def validate_snippet_spdx_id(value, optional=False): + value = value.split('#')[-1] + if re.match(r'^SPDXRef[A-Za-z0-9.\-]+$', value) is not None: + return True + else: + return False + + +def validate_snip_comment(value, optional=False): + return validate_is_free_form_text(value, optional) + + +def validate_snippet_copyright(value, optional=False): + if validate_is_free_form_text(value, optional): + return True + elif isinstance(value, (utils.NoAssert, utils.SPDXNone)): + return True + elif value is None: + return optional + else: + return False + + +def validate_snip_lic_comment(value, optional=False): + return validate_is_free_form_text(value, optional) + + +def validate_snip_file_spdxid(value, optional=False): + if re.match( + r'(DocumentRef[A-Za-z0-9.\-]+:){0,1}SPDXRef[A-Za-z0-9.\-]+', value) is not None: + return True + else: + return False + + +def validate_snip_lics_info(value, optional=False): + if value is None: + return optional + elif isinstance(value, (utils.NoAssert, utils.SPDXNone, document.License)): + return True + else: + return False diff --git a/spdx/snippet.py b/spdx/snippet.py new file mode 100644 index 000000000..033c2d662 --- /dev/null +++ b/spdx/snippet.py @@ -0,0 +1,117 @@ +# Copyright (c) 2018 Yash M. Nisar +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from spdx import document +from spdx import utils + + +class Snippet(object): + """ + Represents an analyzed snippet. + Fields: + - spdx_id: Uniquely identify any element in an SPDX document which may be + referenced by other elements. Mandatory, one per snippet if the snippet + is present. + - name: Name of the snippet. Optional, one. Type: str. + - comment: General comments about the snippet. Optional, one. Type: str. + - copyright: Copyright text. Mandatory, one. Type: str. + - license_comment: Relevant background references or analysis that went + in to arriving at the Concluded License for a snippet. Optional, one. + - snip_from_file_spdxid: Uniquely identify the file in an SPDX document + which this snippet is associated with. Mandatory, one. Type: str. + Type: str. + - conc_lics: Contains the license the SPDX file creator has concluded as + governing the snippet or alternative values if the governing license + cannot be determined. Mandatory one. Type: document.License or + utils.NoAssert or utils.SPDXNone. + - licenses_in_snippet: The list of licenses found in the snippet. + Mandatory, one or more. Type: document.License or utils.SPDXNone or + utils.NoAssert. + """ + + def __init__(self, spdx_id=None, copyright=None, + snip_from_file_spdxid=None, conc_lics=None): + self.spdx_id = spdx_id + self.name = None + self.comment = None + self.copyright = copyright + self.license_comment = None + self.snip_from_file_spdxid = snip_from_file_spdxid + self.conc_lics = conc_lics + self.licenses_in_snippet = [] + + def add_lics(self, lics): + self.licenses_in_snippet.append(lics) + + def validate(self, messages=None): + """ + Validate fields of the snippet and update the messages list with user + friendly error messages for display. + """ + messages = self.validate_spdx_id(messages) + messages = self.validate_copyright_text(messages) + messages = self.validate_snip_from_file_spdxid(messages) + messages = self.validate_concluded_license(messages) + messages = self.validate_licenses_in_snippet(messages) + + return messages + + def validate_spdx_id(self, messages=None): + if self.spdx_id is None: + messages = messages + ['Snippet has no SPDX Identifier.'] + + return messages + + def validate_copyright_text(self, messages=None): + if not isinstance( + self.copyright, + (six.string_types, six.text_type, utils.NoAssert, + utils.SPDXNone)): + messages = messages + [ + 'Snippet copyright must be str or unicode or utils.NoAssert or utils.SPDXNone' + ] + + return messages + + def validate_snip_from_file_spdxid(self, messages=None): + if self.snip_from_file_spdxid is None: + messages = messages + ['Snippet has no Snippet from File SPDX Identifier.'] + + return messages + + def validate_concluded_license(self, messages=None): + if not isinstance(self.conc_lics, (document.License, utils.NoAssert, + utils.SPDXNone)): + messages = messages + [ + 'Snippet Concluded License must be one of ' + 'document.License, utils.NoAssert or utils.SPDXNone' + ] + + return messages + + def validate_licenses_in_snippet(self, messages=None): + if len(self.licenses_in_snippet) == 0: + messages = messages + ['Snippet must have at least one license in file.'] + else: + for lic in self.licenses_in_snippet: + if not isinstance(lic, (document.License, utils.NoAssert, + utils.SPDXNone)): + messages = messages + [ + 'Licenses in Snippet must be one of ' + 'document.License, utils.NoAssert or utils.SPDXNone' + ] + + return messages + + def has_optional_field(self, field): + return getattr(self, field, None) is not None \ No newline at end of file diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index ba60d3124..6e8ff9abf 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -290,6 +290,65 @@ def add_file_dependencies(self): self.add_file_dependencies_helper(doc_file) +class SnippetWriter(LicenseWriter): + + """ + Write spdx.snippet.Snippet + """ + + def __init__(self, document, out): + super(SnippetWriter, self).__init__(document, out) + + def create_snippet_node(self, snippet): + """ + Return a snippet node. + """ + snippet_node = URIRef('http://spdx.org/rdf/terms/Snippet#' + snippet.spdx_id) + type_triple = (snippet_node, RDF.type, self.spdx_namespace.Snippet) + self.graph.add(type_triple) + + if snippet.has_optional_field('comment'): + comment_triple = (snippet_node, RDFS.comment, Literal(snippet.comment)) + self.graph.add(comment_triple) + + if snippet.has_optional_field('name'): + name_triple = (snippet_node, self.spdx_namespace.name, Literal(snippet.name)) + self.graph.add(name_triple) + + if snippet.has_optional_field('license_comment'): + lic_comment_triple = (snippet_node, self.spdx_namespace.licenseComments, + Literal(snippet.license_comment)) + self.graph.add(lic_comment_triple) + + cr_text_node = self.to_special_value(snippet.copyright) + cr_text_triple = (snippet_node, self.spdx_namespace.copyrightText, cr_text_node) + self.graph.add(cr_text_triple) + + snip_from_file_triple = (snippet_node, self.spdx_namespace.snippetFromFile, + Literal(snippet.snip_from_file_spdxid)) + self.graph.add(snip_from_file_triple) + + conc_lic_node = self.license_or_special(snippet.conc_lics) + conc_lic_triple = ( + snippet_node, self.spdx_namespace.licenseConcluded, conc_lic_node) + self.graph.add(conc_lic_triple) + + license_info_nodes = map(self.license_or_special, + snippet.licenses_in_snippet) + for lic in license_info_nodes: + triple = ( + snippet_node, self.spdx_namespace.licenseInfoInSnippet, lic) + self.graph.add(triple) + + return snippet_node + + def snippets(self): + """ + Return list of snippet nodes. + """ + return map(self.create_snippet_node, self.document.snippet) + + class ReviewInfoWriter(BaseWriter): """ @@ -576,7 +635,7 @@ def handle_package_has_file(self, package, package_node): class Writer(CreationInfoWriter, ReviewInfoWriter, FileWriter, PackageWriter, - ExternalDocumentRefWriter, AnnotationInfoWriter): + ExternalDocumentRefWriter, AnnotationInfoWriter, SnippetWriter): """ Warpper for other writers to write all fields of spdx.document.Document Call `write()` to start writing. @@ -637,6 +696,10 @@ def write(self): package_node = self.packages() package_triple = (doc_node, self.spdx_namespace.describesPackage, package_node) self.graph.add(package_triple) + # Add snippet + snippet_nodes = self.snippets() + for snippet in snippet_nodes: + self.graph.add((doc_node, self.spdx_namespace.Snippet, snippet)) # normalize the graph to ensure that the sort order is stable self.graph = to_isomorphic(self.graph) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index f48b851f3..a0942432d 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -149,6 +149,31 @@ def write_file(spdx_file, out): write_value('ArtifactOfProjectURI', uri, out) +def write_snippet(snippet, out): + """ + Write snippet fields to out. + """ + out.write('# Snippet\n\n') + write_value('SnippetSPDXID', snippet.spdx_id, out) + write_value('SnippetFromFileSPDXID', snippet.snip_from_file_spdxid, out) + write_text_value('SnippetCopyrightText', snippet.copyright, out) + if snippet.has_optional_field('name'): + write_value('SnippetName', snippet.name, out) + if snippet.has_optional_field('comment'): + write_text_value('SnippetComment', snippet.comment, out) + if snippet.has_optional_field('license_comment'): + write_text_value('SnippetLicenseComments', snippet.license_comment, out) + if isinstance(snippet.conc_lics, + (document.LicenseConjunction, document.LicenseDisjunction)): + write_value('SnippetLicenseConcluded', u'({0})'.format( + snippet.conc_lics), out) + else: + write_value('SnippetLicenseConcluded', snippet.conc_lics, out) + # Write sorted list + for lics in sorted(snippet.licenses_in_snippet): + write_value('LicenseInfoInSnippet', lics, out) + + def write_package(package, out): """ Write a package fields to out. @@ -281,6 +306,11 @@ def write_document(document, out, validate=True): write_package(document.package, out) write_separators(out) + # Write out snippet info + for snippet in document.snippet: + write_snippet(snippet, out) + write_separators(out) + out.write('# Extracted Licenses\n\n') for lic in sorted(document.extracted_licenses): write_extracted_licenses(lic, out) diff --git a/tests/test_builder.py b/tests/test_builder.py index cb83c0c13..81c025d4d 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -508,5 +508,143 @@ def test_set_pkg_desc_order(self): self.builder.set_pkg_desc(self.document, 'something') +class TestSnippetBuilder(TestCase): + + def setUp(self): + self.entity_builder = builders.EntityBuilder() + self.builder = builders.SnippetBuilder() + self.document = Document() + + def test_create_snippet(self): + assert self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + + @testing_utils.raises(builders.SPDXValueError) + def test_incorrect_snippet_spdx_id(self): + self.builder.create_snippet(self.document, 'Some_value_with_$%') + + def test_snippet_name(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_name(self.document, 'Name_of_snippet') + + @testing_utils.raises(builders.OrderError) + def test_snippet_name_order(self): + self.builder.set_snippet_name(self.document, 'Name_of_snippet') + + def test_snippet_comment(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_comment(self.document, 'Comment') + + @testing_utils.raises(builders.OrderError) + def test_snippet_comment_order(self): + self.builder.set_snippet_comment(self.document, 'Comment') + + @testing_utils.raises(builders.SPDXValueError) + def test_snippet_comment_text_value(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_comment(self.document, 'Comment.') + + def test_snippet_copyright(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_copyright(self.document, 'Copyright 2008-2010 John Smith') + + @testing_utils.raises(builders.SPDXValueError) + def test_snippet_copyright_text_value(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_copyright(self.document, + 'Copyright 2008-2010 John Smith') + + @testing_utils.raises(builders.OrderError) + def test_snippet_copyright_order(self): + self.builder.set_snippet_copyright(self.document, + 'Copyright 2008-2010 John Smith') + + def test_snippet_lic_comment(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_lic_comment(self.document, + 'Lic comment') + + @testing_utils.raises(builders.SPDXValueError) + def test_snippet_lic_comment_text_value(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_lic_comment(self.document, + 'Lic comment') + + @testing_utils.raises(builders.OrderError) + def test_snippet_lic_comment_order(self): + self.builder.set_snippet_lic_comment(self.document, + 'Lic comment') + + def test_snippet_from_file_spdxid(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snip_from_file_spdxid(self.document, + 'SPDXRef-DoapSource') + + @testing_utils.raises(builders.SPDXValueError) + def test_snippet_from_file_spdxid_value(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snip_from_file_spdxid(self.document, + '#_$random_chars') + + @testing_utils.raises(builders.OrderError) + def test_snippet_from_file_spdxid_order(self): + self.builder.set_snip_from_file_spdxid(self.document, + 'SPDXRef-DoapSource') + + @testing_utils.raises(builders.CardinalityError) + def test_snippet_from_file_spdxid_cardinality(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snip_from_file_spdxid(self.document, + 'SPDXRef-DoapSource') + self.builder.set_snip_from_file_spdxid(self.document, + 'SPDXRef-somevalue') + + def test_snippet_conc_lics(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snip_concluded_license(self.document, + License.from_identifier( + 'Apache-2.0')) + + @testing_utils.raises(builders.SPDXValueError) + def test_snippet_conc_lics_value(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snip_concluded_license(self.document, 'Apache-2.0') + + @testing_utils.raises(builders.OrderError) + def test_snippet_conc_lics_order(self): + self.builder.set_snip_concluded_license(self.document, + License.from_identifier( + 'Apache-2.0')) + + @testing_utils.raises(builders.CardinalityError) + def test_snippet_conc_lics_cardinality(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snip_concluded_license(self.document, + License.from_identifier( + 'Apache-2.0')) + self.builder.set_snip_concluded_license(self.document, + License.from_identifier( + 'Apache-2.0')) + + def test_snippet_lics_info(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_lics_info(self.document, + License.from_identifier( + 'Apache-2.0')) + self.builder.set_snippet_lics_info(self.document, + License.from_identifier( + 'GPL-2.0-or-later')) + + @testing_utils.raises(builders.SPDXValueError) + def test_snippet_lics_info_value(self): + self.builder.create_snippet(self.document, 'SPDXRef-Snippet') + self.builder.set_snippet_lics_info(self.document, 'Apache-2.0') + + @testing_utils.raises(builders.OrderError) + def test_snippet_lics_info_order(self): + self.builder.set_snippet_lics_info(self.document, + License.from_identifier( + 'Apache-2.0')) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_tag_value_parser.py b/tests/test_tag_value_parser.py index 6f2f06fa9..61dc616f6 100644 --- a/tests/test_tag_value_parser.py +++ b/tests/test_tag_value_parser.py @@ -133,6 +133,38 @@ def test_unknown_tag(self): self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG', 'SomeUnknownTag', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2) + def test_snippet(self): + data = ''' + SnippetSPDXID: SPDXRef-Snippet + SnippetLicenseComments: Some lic comment. + SnippetCopyrightText: Some cr text. + SnippetComment: Some snippet comment. + SnippetName: from linux kernel + SnippetFromFileSPDXID: SPDXRef-DoapSource + SnippetLicenseConcluded: Apache-2.0 + LicenseInfoInSnippet: Apache-2.0 + ''' + self.l.input(data) + self.token_assert_helper(self.l.token(), 'SNIPPET_SPDX_ID', 'SnippetSPDXID', 2) + self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Snippet', 2) + self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_COMMENT', 'SnippetLicenseComments', 3) + self.token_assert_helper(self.l.token(), 'TEXT', 'Some lic comment.', 3) + self.token_assert_helper(self.l.token(), 'SNIPPET_CR_TEXT', 'SnippetCopyrightText', 4) + self.token_assert_helper(self.l.token(), 'TEXT', 'Some cr text.', 4) + self.token_assert_helper(self.l.token(), 'SNIPPET_COMMENT', 'SnippetComment', 5) + self.token_assert_helper(self.l.token(), 'TEXT', 'Some snippet comment.', 5) + self.token_assert_helper(self.l.token(), 'SNIPPET_NAME', 'SnippetName', 6) + self.token_assert_helper(self.l.token(), 'LINE', 'from linux kernel', 6) + self.token_assert_helper(self.l.token(), 'SNIPPET_FILE_SPDXID', + 'SnippetFromFileSPDXID', 7) + self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DoapSource', 7) + self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_CONC', + 'SnippetLicenseConcluded', 8) + self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 8) + self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_INFO', + 'LicenseInfoInSnippet', 9) + self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 9) + def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value @@ -202,7 +234,18 @@ class TestParser(TestCase): unknown_tag_str = 'SomeUnknownTag: SomeUnknownValue' - complete_str = '{0}\n{1}\n{2}\n{3}\n{4}'.format(document_str, creation_str, review_str, package_str, file_str) + snippet_str = '\n'.join([ + 'SnippetSPDXID: SPDXRef-Snippet', + 'SnippetLicenseComments: Some lic comment.', + 'SnippetCopyrightText: Copyright 2008-2010 John Smith ', + 'SnippetComment: Some snippet comment.', + 'SnippetName: from linux kernel', + 'SnippetFromFileSPDXID: SPDXRef-DoapSource', + 'SnippetLicenseConcluded: Apache-2.0', + 'LicenseInfoInSnippet: Apache-2.0', + ]) + + complete_str = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format(document_str, creation_str, review_str, package_str, file_str, snippet_str) def setUp(self): self.p = Parser(Builder(), StandardLogger()) @@ -270,6 +313,20 @@ def test_unknown_tag(self): assert error assert document is not None + def test_snippet(self): + document, error = self.p.parse(self.complete_str) + assert document is not None + assert not error + assert len(document.snippet) == 1 + assert document.snippet[-1].spdx_id == 'SPDXRef-Snippet' + assert document.snippet[-1].name == 'from linux kernel' + assert document.snippet[-1].comment == 'Some snippet comment.' + assert document.snippet[-1].copyright == ' Copyright 2008-2010 John Smith ' + assert document.snippet[-1].license_comment == 'Some lic comment.' + assert document.snippet[-1].snip_from_file_spdxid == 'SPDXRef-DoapSource' + assert document.snippet[-1].conc_lics.identifier == 'Apache-2.0' + assert document.snippet[-1].licenses_in_snippet[-1].identifier == 'Apache-2.0' + if __name__ == '__main__': unittest.main()