spdx_tools.spdx.parser.tagvalue.lexer

  1# Copyright (c) 2014 Ahmed H. Ismail
  2# Copyright (c) 2023 spdx contributors
  3# SPDX-License-Identifier: Apache-2.0
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#     http://www.apache.org/licenses/LICENSE-2.0
  8# Unless required by applicable law or agreed to in writing, software
  9# distributed under the License is distributed on an "AS IS" BASIS,
 10# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 11# See the License for the specific language governing permissions and
 12# limitations under the License.
 13
 14from ply import lex
 15from ply.lex import TOKEN
 16
 17
 18class SPDXLexer:
 19    reserved = {
 20        # Top level fields
 21        "SPDXVersion": "DOC_VERSION",
 22        "DataLicense": "DOC_LICENSE",
 23        "DocumentName": "DOC_NAME",
 24        "SPDXID": "SPDX_ID",
 25        "DocumentComment": "DOC_COMMENT",
 26        "DocumentNamespace": "DOC_NAMESPACE",
 27        "ExternalDocumentRef": "EXT_DOC_REF",
 28        # Creation info fields
 29        "Creator": "CREATOR",
 30        "Created": "CREATED",
 31        "CreatorComment": "CREATOR_COMMENT",
 32        "LicenseListVersion": "LICENSE_LIST_VERSION",
 33        # Annotation fields
 34        "Annotator": "ANNOTATOR",
 35        "AnnotationDate": "ANNOTATION_DATE",
 36        "AnnotationComment": "ANNOTATION_COMMENT",
 37        "AnnotationType": "ANNOTATION_TYPE",
 38        "SPDXREF": "ANNOTATION_SPDX_ID",
 39        # Relationship fields
 40        "Relationship": "RELATIONSHIP",
 41        "RelationshipComment": "RELATIONSHIP_COMMENT",
 42        # Package fields
 43        "PackageName": "PKG_NAME",
 44        "PackageVersion": "PKG_VERSION",
 45        "PackageDownloadLocation": "PKG_DOWNLOAD_LOCATION",
 46        "FilesAnalyzed": "PKG_FILES_ANALYZED",
 47        "PackageSummary": "PKG_SUMMARY",
 48        "PackageSourceInfo": "PKG_SOURCE_INFO",
 49        "PackageFileName": "PKG_FILE_NAME",
 50        "PackageSupplier": "PKG_SUPPLIER",
 51        "PackageOriginator": "PKG_ORIGINATOR",
 52        "PackageChecksum": "PKG_CHECKSUM",
 53        "PackageVerificationCode": "PKG_VERIFICATION_CODE",
 54        "PackageDescription": "PKG_DESCRIPTION",
 55        "PackageComment": "PKG_COMMENT",
 56        "PackageLicenseDeclared": "PKG_LICENSE_DECLARED",
 57        "PackageLicenseConcluded": "PKG_LICENSE_CONCLUDED",
 58        "PackageLicenseInfoFromFiles": "PKG_LICENSE_INFO",
 59        "PackageLicenseComments": "PKG_LICENSE_COMMENT",
 60        "PackageCopyrightText": "PKG_COPYRIGHT_TEXT",
 61        "PackageHomePage": "PKG_HOMEPAGE",
 62        "ExternalRef": "PKG_EXTERNAL_REF",
 63        "ExternalRefComment": "PKG_EXTERNAL_REF_COMMENT",
 64        "PackageAttributionText": "PKG_ATTRIBUTION_TEXT",
 65        "PrimaryPackagePurpose": "PRIMARY_PACKAGE_PURPOSE",
 66        "BuiltDate": "BUILT_DATE",
 67        "ReleaseDate": "RELEASE_DATE",
 68        "ValidUntilDate": "VALID_UNTIL_DATE",
 69        # File fields
 70        "FileName": "FILE_NAME",
 71        "FileType": "FILE_TYPE",
 72        "FileChecksum": "FILE_CHECKSUM",
 73        "LicenseConcluded": "FILE_LICENSE_CONCLUDED",
 74        "LicenseInfoInFile": "FILE_LICENSE_INFO",
 75        "FileCopyrightText": "FILE_COPYRIGHT_TEXT",
 76        "LicenseComments": "FILE_LICENSE_COMMENT",
 77        "FileComment": "FILE_COMMENT",
 78        "FileNotice": "FILE_NOTICE",
 79        "FileContributor": "FILE_CONTRIBUTOR",
 80        "FileAttributionText": "FILE_ATTRIBUTION_TEXT",
 81        # ExtractedLicensingInfo fields
 82        "LicenseID": "LICENSE_ID",
 83        "ExtractedText": "LICENSE_TEXT",
 84        "LicenseName": "LICENSE_NAME",
 85        "LicenseCrossReference": "LICENSE_CROSS_REF",
 86        "LicenseComment": "LICENSE_COMMENT",
 87        # Snippet fields
 88        "SnippetSPDXID": "SNIPPET_SPDX_ID",
 89        "SnippetName": "SNIPPET_NAME",
 90        "SnippetComment": "SNIPPET_COMMENT",
 91        "SnippetCopyrightText": "SNIPPET_COPYRIGHT_TEXT",
 92        "SnippetLicenseComments": "SNIPPET_LICENSE_COMMENT",
 93        "SnippetFromFileSPDXID": "SNIPPET_FILE_SPDXID",
 94        "SnippetLicenseConcluded": "SNIPPET_LICENSE_CONCLUDED",
 95        "LicenseInfoInSnippet": "SNIPPET_LICENSE_INFO",
 96        "SnippetAttributionText": "SNIPPET_ATTRIBUTION_TEXT",
 97        "SnippetByteRange": "SNIPPET_BYTE_RANGE",
 98        "SnippetLineRange": "SNIPPET_LINE_RANGE",
 99        # Common fields
100        "NOASSERTION": "NO_ASSERTION",
101        "NONE": "NONE",
102    }
103    states = (("text", "exclusive"),)
104
105    tokens = [
106        "TEXT",
107        "TOOL_VALUE",
108        "UNKNOWN_TAG",
109        "ORGANIZATION_VALUE",
110        "PERSON_VALUE",
111        "ISO8601_DATE",
112        "LINE",
113        "CHECKSUM",
114    ] + list(reserved.values())
115
116    def __init__(self):
117        self.lexer = None
118
119    @TOKEN(r":\s*<text>")
120    def t_text(self, t):
121        t.lexer.text_start = t.lexer.lexpos - len("<text>")
122        t.lexer.begin("text")
123
124    @TOKEN(r"</text>\s*")
125    def t_text_end(self, t):
126        t.type = "TEXT"
127        t.value = t.lexer.lexdata[t.lexer.text_start : t.lexer.lexpos]
128        t.lexer.lineno += t.value.count("\n")
129        t.value = t.value.strip()
130        t.lexer.begin("INITIAL")
131        return t
132
133    @TOKEN(r".|\n")
134    def t_text_any(self, t):
135        pass
136
137    def t_text_error(self, t):
138        print("Lexer error in text state")
139
140    @TOKEN(
141        r":\s*(ADLER32|BLAKE2b-256|BLAKE2b-384|BLAKE2b-512|BLAKE3|MD2|MD4|MD5|MD6|SHA1|SHA224|SHA256|SHA384|SHA512|"
142        r"SHA3-256|SHA3-384|SHA3-512):\s*([a-f0-9]*)"
143    )
144    def t_CHECKSUM(self, t):
145        t.value = t.value[1:].strip()
146        return t
147
148    @TOKEN(r":\s*Tool:.+")
149    def t_TOOL_VALUE(self, t):
150        t.value = t.value[1:].strip()
151        return t
152
153    @TOKEN(r":\s*Organization:.+")
154    def t_ORGANIZATION_VALUE(self, t):
155        t.value = t.value[1:].strip()
156        return t
157
158    @TOKEN(r":\s*Person:.+")
159    def t_PERSON_VALUE(self, t):
160        t.value = t.value[1:].strip()
161        return t
162
163    @TOKEN(r":\s*\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ")
164    def t_ISO8601_DATE(self, t):
165        t.value = t.value[1:].strip()
166        return t
167
168    @TOKEN(r"[a-zA-Z]+")
169    def t_KEYWORD_AS_TAG(self, t):
170        t.type = self.reserved.get(t.value, "UNKNOWN_TAG")
171        t.value = t.value.strip()
172        return t
173
174    @TOKEN(r":.+")
175    def t_LINE_OR_KEYWORD_VALUE(self, t):
176        t.value = t.value[1:].strip()
177        if t.value in self.reserved.keys():
178            t.type = self.reserved[t.value]
179        else:
180            t.type = "LINE"
181        return t
182
183    @TOKEN(r"\#.*")
184    def t_comment(self, t):
185        pass
186
187    @TOKEN(r"\n+")
188    def t_newline(self, t):
189        t.lexer.lineno += len(t.value)
190
191    @TOKEN(r"[ \t]+")
192    def t_whitespace(self, t):
193        pass
194
195    def build(self, **kwargs):
196        self.lexer = lex.lex(module=self, **kwargs)
197
198    def token(self):
199        return self.lexer.token()
200
201    def input(self, data):
202        self.lexer.input(data)
203
204    def t_error(self, t):
205        t.lexer.skip(1)
206        t.value = "Lexer error"
207        return t