Merge pull request #323 from liudmylaru/add_precommit_hooks

Add pre-commit hooks for ocids, copyright and secrets

Author: darenr

.gitleaks.toml

@@ -0,0 +1,50 @@
+title = "Gitleaks Config"
+
+# Gitleaks feature, extending the existing base config from:
+# https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml
+[extend]
+useDefault = true
+
+# Allowlist's 'stopwords' and 'regexes' excludes any secrets or mathching patterns from the current repository.
+# Paths listed in allowlist will not be scanned.
+[allowlist]
+    description = "Global allow list"
+    stopwords = ["test_password", "sample_key"]
+    regexes = [
+        '''example-password''',
+        '''this-is-not-the-secret''',
+        '''<redacted>'''
+    ]
+    paths = [
+        '''^(actions|ai_services|data|distributed|jobs|labs|model|notebook_lifecycle|pipelines|use_cases)''',
+        '''^(.git|.pre-commit)''',
+        '''CODE_OF_CONDUCT.md''',
+        '''CONTRIBUTING.md''',
+        '''LICENSE.txt''',
+        '''package.json''',
+        '''^README.md''',
+        '''SECURITY.md''',
+        '''THIRD_PARTY_LICENSES.TXT''',
+    ]
+
+# Describe rule to search real ocids
+[[rules]]
+    description = "Real ocids"
+    id = "ocid"
+    path = '''notebook_examples'''
+    regex = '''ocid[123]\.[a-z1-9A-Z]*\.oc\d\.[a-z1-9A-Z]*\.[a-z1-9A-Z]+'''
+    keywords = [
+        "ocid"
+    ]
+
+# Describe rule to search generic secrets
+[[rules]]
+    description = "Generic secret"
+    id = "generic-secret"
+    path = '''notebook_examples'''
+    regex = '''(?i)((key|api|token|secret|passwd|password|psw|pass|pswd)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z!@#$%^&*<>\\\-_.=]{3,100})['\"]'''
+    entropy = 0
+    secretGroup = 4
+    keywords = [
+        "key","api","token","secret","passwd","password", "psw", "pass", "pswd"
+    ]

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+# To use:
+#
+#     pre-commit run --all-files  # run all hooks on all files
+#     pre-commit run <HOOK_ID> --all-files  # run one hooks on all files
+#     pre-commit run --files <path_to_file> <path_to_folder/**>  # run all hook on files
+#     pre-commit run <HOOK_ID> --files <path_to_file> <path_to_folder/**>  # run one hook on files
+#
+# Or:
+#
+#     pre-commit install  # (runs every time you commit in git)
+#
+# To update this file:
+#
+#     pre-commit autoupdate
+#
+# See https://github.com/pre-commit/pre-commit
+
+# Detect hardcoded secrets and ocids in notebook_examples/ folder
+repos:
+-   repo: https://github.com/gitleaks/gitleaks
+    rev: v8.17.0
+    hooks:
+    -   id: gitleaks
+        files: ^notebook_examples/
+# Oracle copyright checkers in notebook_examples/ folder
+-   repo: local
+    hooks:
+    -   id: check-copyright
+        name: check-copyright
+        entry: .pre-commit-scripts/check-copyright.py
+        language: script
+        types_or: ['python', 'shell', 'bash']
+        files: ^notebook_examples/

.pre-commit-hooks.yaml

@@ -0,0 +1,6 @@
+-  id: check-copyright
+   name: check-copyright
+   description: Validate Oracle copyright and license statements
+   entry: .pre-commit-scripts/check-copyright.py
+   language: script
+   types_or: ['python', 'shell', 'bash']

.pre-commit-scripts/check-copyright.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import datetime
+import os
+import sys
+
+CURRENT_YEAR = datetime.date.today().year
+
+### At the beginning of next year line has to be added in this list:
+PUBLISHED_LAST_EDITED_YEARS = [
+    f"Copyright (c) 2020, {CURRENT_YEAR} Oracle and/or its affiliates",
+    f"Copyright (c) 2021, {CURRENT_YEAR} Oracle and/or its affiliates",
+    f"Copyright (c) 2022, {CURRENT_YEAR} Oracle and/or its affiliates",
+    f"Copyright (c) {CURRENT_YEAR} Oracle and/or its affiliates",
+]
+
+LICENSE_STATEMENTS = [
+    "Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/"
+]
+
+def main(filenames) -> int:
+    phrases = LICENSE_STATEMENTS
+    years = PUBLISHED_LAST_EDITED_YEARS
+    retcode = 0
+    for filename in filenames:
+        if not os.path.basename(filename).startswith("."):
+            with open(filename) as inputfile:
+                content = inputfile.read()
+                if not any(x in content for x in years):
+                    print(f"{filename}: Year published or year last edited not correct.")
+                    retcode = 1
+                    break
+                for p in phrases:
+                    if p not in content:
+                        print(f"{filename}: Copyright text missing or incomplete.")
+                        retcode = 1
+                        break
+
+    sys.exit(retcode)
+
+
+if __name__ == "__main__":
+    filenames = sys.argv
+    main(filenames)