adjustments in redaction code

README.md

@@ -164,7 +164,7 @@ First, you need to import the **bravado** library for **validate_object**.
 
 ![import-bravado.png](images/import-bravado.png)
 
-This code is responsible to redact your validation response:
+This code is responsible to beautify your validation response:
 
 ![redact-code.png](images/redact-code.png)
 
@@ -174,10 +174,23 @@ And you can put in the main function code:
 
 ![main-redact-routine.png](images/main-redact-routine.png)
 
-The response will be like this:
+You need to use a library called [Redaction.py](./files/authApi/Redaction.py), included in this material. This class find your patterns for attributes that need redaction.
 
-![redact-response.png](images/redact-response.png)
+Declare your patterns and attribute names:
 
+![redaction-2.png](images/redaction-2.png)
+
+So you can use the library like this:
+
+![redaction-1.png](images/redaction-1.png)
+
+When you request an REST service with this body content:
+
+![img.png](images/redaction-4.png)
+
+The response will be something like this:
+
+![redaction-3.png](images/redaction-3.png)
 
 ## Resource Principal
 

files/authApi/Redaction.py

@@ -0,0 +1,169 @@
+import re
+
+# This class does not work with numeric attributes, only with String
+# Use the method redact and put your patterns as the example:
+# SENSITIVE_PATTERNS = [
+#     r"\d{3}-\d{2}-\d{4}",  # Social Security Number (SSN) pattern
+#     r"\d{4}[-\s]\d{4}[-\s]\d{4}[-\s]\d{4}",  # Credit card number pattern
+#     r"\(?\d{3}\)?[-\s.]?\d{3}[-\s.]?\d{4}",  # Phone number
+#     r"(0[1-9]|1[0-2])[-/.](0[1-9]|[12][0-9]|3[01])[-/.](19|20)\d\d",  # date of birth
+#     r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)",  # IP address
+#     r"[a-zA-Z0-9]{32}"  # API key
+# ]
+#
+# ATTRIBUTE_PATTERNS = [
+#     "nome",
+#     "cpf",
+#     "teste",
+#     "valor",
+#     "original",
+#     "type",
+#     "solicitacaoPagador",
+#     "chave",
+#     "description",
+#     "items",
+#     "example"
+# ]
+
+class Redaction():
+
+    def repl_value2(self, message, pattern):
+        flag_aspas = False
+        flag_attribute = False
+        # flag_vezes = 0
+        flag_dois_pontos = False
+        flag_colchetes = False
+        i = 0
+        z = pattern
+        str_acc = ""
+        while (i < len(message)):
+            try:
+                if (message[i:i + len(z)] == z and (message[i + len(z):i + len(z) + 1] == "'" or message[i + len(z):i + len(z) + 1] == "\"")):
+                    flag_attribute = True
+                    flag_aspas = True
+            except:
+                print("except")
+            if (message[i] == ":" and not flag_aspas and flag_attribute):
+                flag_dois_pontos = True
+            if (flag_aspas and message[i] != "'" and message[i] != "\"" and flag_attribute and flag_dois_pontos):
+                str_acc = str_acc + message[i]
+                message = message[0:i] + "*" + message[i + 1:len(message) + i]
+            if (message[i] == "{" and flag_dois_pontos and not flag_aspas):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+            if ((message[i] == "}" or message[i] == "]") and not flag_aspas):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+                str_acc = ""
+            if (flag_dois_pontos and not flag_aspas and message[i] == "["):
+                flag_colchetes = True
+            if (message[i] == "," and not flag_aspas and not flag_colchetes):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+                str_acc = ""
+            if ((message[i] == "'" or message[i] == "\"")):
+                flag_aspas = not flag_aspas
+            if (flag_aspas == False and flag_attribute == True and flag_dois_pontos and len(str_acc) > 0 and not flag_colchetes):
+                flag_attribute = False
+                flag_dois_pontos = False
+                str_acc = ""
+            i = i + 1
+        return message
+
+    def repl_value(self, message, pattern):
+        flag_aspas = False
+        flag_attribute = False
+        # flag_vezes = 0
+        flag_dois_pontos = False
+        flag_colchetes = False
+        flag_string = True
+        flag_descobre_tipo = False
+        i = 0
+        z = pattern
+        str_acc = ""
+        while (i < len(message)):
+            try:
+                if (message[i:i + len(z)] == z and (message[i + len(z):i + len(z) + 1] == "'" or message[i + len(z):i + len(z) + 1] == "\"")):
+                    flag_attribute = True
+                    flag_aspas = True
+            except:
+                print("except")
+            if (message[i] == ":" and not flag_aspas and flag_attribute):
+                flag_dois_pontos = True
+                flag_descobre_tipo = True
+            if ((flag_aspas and message[i] != "'" and message[i] != "\"" and flag_attribute and flag_dois_pontos)
+                    or (message[i] in "0123456789." and flag_attribute and flag_dois_pontos and not flag_string)):
+                str_acc = str_acc + message[i]
+                message = message[0:i] + "*" + message[i + 1:len(message) + i]
+            if (message[i] == "{" and flag_dois_pontos and not flag_aspas):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+                flag_descobre_tipo = False
+                flag_string = True
+            if ((message[i] == "}" or message[i] == "]") and not flag_aspas):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+                flag_descobre_tipo = False
+                flag_string = True
+                str_acc = ""
+            if ((message[i] == "}" or message[i] == "]") and not flag_aspas and not flag_string):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+                flag_descobre_tipo = False
+                flag_string = True
+                str_acc = ""
+            if (flag_dois_pontos and not flag_aspas and message[i] == "["):
+                flag_colchetes = True
+            if (message[i] == "," and not flag_aspas and not flag_colchetes and not flag_string):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_aspas = False
+                flag_colchetes = False
+                flag_descobre_tipo = False
+                flag_string = True
+                str_acc = ""
+            if ((message[i] == "'" or message[i] == "\"")):
+                flag_aspas = not flag_aspas
+                if (flag_descobre_tipo):
+                    flag_string = True
+                    flag_descobre_tipo = False
+            if (message[i] in "01234567890." and flag_descobre_tipo):
+                flag_string = False
+                flag_descobre_tipo = False
+                str_acc = str_acc + message[i]
+                message = message[0:i] + "*" + message[i + 1:len(message) + i]
+            if (flag_aspas == False and flag_attribute == True and flag_dois_pontos and len(str_acc) > 0 and not flag_colchetes and flag_string):
+                flag_attribute = False
+                flag_dois_pontos = False
+                flag_descobre_tipo = False
+                str_acc = ""
+            i = i + 1
+        return message
+
+    def repl(self, attribute_pattern, message):
+        msg_return = []
+        for pattern in attribute_pattern:
+            message = self.repl_value(message, pattern)
+        return message
+
+    def change(self, sensitive_pattern, message):
+        for pattern in sensitive_pattern:
+            message = re.sub(pattern, "<REDACTED>", message)
+        return message
+
+    def redact(self, sensitive_pattern, attribute_pattern, message):
+        message = self.repl(attribute_pattern, message)
+        message = self.change(sensitive_pattern, message)
+        return message

files/authApi/authRPApi.py

@@ -10,6 +10,43 @@ import os
 import ast
 from bravado_core.spec import Spec
 from bravado_core.validate import validate_object
+from datetime import datetime
+from random import randrange
+
+import Redaction
+
+SENSITIVE_PATTERNS = [
+    r"\d{3}-\d{2}-\d{4}",  # Social Security Number (SSN) pattern
+    r"\d{4}[-\s]\d{4}[-\s]\d{4}[-\s]\d{4}",  # Credit card number pattern
+    r"\(?\d{3}\)?[-\s.]?\d{3}[-\s.]?\d{4}",  # Phone number
+    r"(0[1-9]|1[0-2])[-/.](0[1-9]|[12][0-9]|3[01])[-/.](19|20)\d\d",  # date of birth
+    r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)",  # IP address
+    r"[a-zA-Z0-9]{32}",  # API key
+    r"^(\d{5}.\d{2}-\d)|(\d{8})$"
+]
+
+ATTRIBUTE_PATTERNS = [
+    "documentNumber",
+    "documentCustodyAgentAccountCode",
+    "isinCode",
+    "payingAgentAccountCode",
+    "registrationParticipantAccountCode",
+    "nome",
+    "$ref",
+    "cpf",
+    "teste",
+    "valor",
+    "original",
+    "type",
+    "solicitacaoPagador",
+    "expiracao",
+    "chave",
+    "description",
+    "items",
+    "required",
+    "x-scope",
+    "maxLength"
+]
 
 #### IDCS Routines
 #### https://docs.oracle.com/en/learn/apigw-modeldeployment/index.html#introduction
@@ -51,20 +88,6 @@ def replace_regex(variavel):
 
     return variavel
 
-def replace_escape_chars(obj):
-    for k, v in obj.items():
-        if isinstance(v, str):
-            obj[k] = replace_regex(v)
-        elif isinstance(v, dict):
-            obj[k] = replace_escape_chars(v)
-        elif isinstance(v, list):
-            for i in range(len(v)):
-                if isinstance(v[i], str):
-                    v[i] = replace_regex(v[i])
-                elif isinstance(v[i], dict):
-                    v[i] = replace_escape_chars(v[i])
-    return obj
-
 def remove_property(dictionary, property_name):
     keys_to_delete = [key for key in dictionary if key == property_name]
     for key in keys_to_delete:
@@ -82,6 +105,14 @@ def remove_property(dictionary, property_name):
                     remove_property(item, property_name)
     return dictionary
 
+def count_attributes(json_data):
+    count = 0
+    for key, value in json_data.items():
+        count += 1
+        if isinstance(value, dict):
+            count += count_attributes(value)
+    return count
+
 def read_secret_value(secret_client, secret_id):
     response = secret_client.get_secret_bundle(secret_id)
     base64_Secret_content = response.data.secret_bundle_content.content
@@ -91,9 +122,8 @@ def read_secret_value(secret_client, secret_id):
     return secret_content
 
 def handler(ctx, data: io.BytesIO = None):
-    signer = oci.auth.signers.get_resource_principals_signer()
+    config = oci.config.from_file("config")
-    logging = oci.loggingingestion.LoggingClient(config={}, signer=signer)
+    logging = oci.loggingingestion.LoggingClient(config)
-    secret_client = oci.secrets.SecretsClient(config={}, signer=signer)
 
     # functions context variables
     app_context = dict(ctx.Config())
@@ -108,6 +138,25 @@ def handler(ctx, data: io.BytesIO = None):
         ClientId = read_secret_value(secret_client, "ocid1.vaultsecret.oc1.iad.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
         ClientSecret = read_secret_value(secret_client, "ocid1.vaultsecret.oc1.iad.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
 
+        # JSON Items counter
+        jsonData = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"]
+        jsonData = dict(json.loads(jsonData))
+        c = count_attributes(jsonData)
+        if (c > 21):
+            rdata = json.dumps({
+                "active": False,
+                "context": {
+                    "status_code": 401,
+                    "message": "JSON exception",
+                    "error": "JSON exception",
+                }})
+
+            return response.Response(
+                ctx,
+                status_code=401,
+                response_data=rdata
+            )
+
         try:
             body = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"]
             body = json.loads(body)
@@ -184,6 +233,8 @@ def handler(ctx, data: io.BytesIO = None):
                     )
                 except(Exception) as ex2:
                     error_msg = beautify_str(str(ex2))
+                    redaction = Redaction.Redaction()
+                    error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg)
                     put_logs_response = logging.put_logs(
                         log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan",
                         put_logs_details=oci.loggingingestion.models.PutLogsDetails(
@@ -225,15 +276,20 @@ def handler(ctx, data: io.BytesIO = None):
                     api_spec = apigateway_client.get_api_content(contents[1])
                     spec_dict = json.loads(api_spec.data.content)
                     spec_dict = remove_property(spec_dict, "pattern")
+
                     spec = Spec.from_dict(spec_dict, config=bravado_config)
                     try:
                         schema = spec_dict["definitions"][contents[0]]
                     except:
                         schema = spec_dict["components"]["schemas"][contents[0]]
+
                     schema_without_pattern = remove_property(schema, "pattern")
+
                     validate_object(spec, schema_without_pattern, body)
                 except (Exception) as ex3:
                     error_msg = beautify_str(str(ex3))
+                    redaction = Redaction.Redaction()
+                    error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg)
                     put_logs_response = logging.put_logs(
                         log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan",
                         put_logs_details=oci.loggingingestion.models.PutLogsDetails(

files/authApi/func.py

@@ -10,6 +10,43 @@ import os
 import ast
 from bravado_core.spec import Spec
 from bravado_core.validate import validate_object
+from datetime import datetime
+from random import randrange
+
+import Redaction
+
+SENSITIVE_PATTERNS = [
+    r"\d{3}-\d{2}-\d{4}",  # Social Security Number (SSN) pattern
+    r"\d{4}[-\s]\d{4}[-\s]\d{4}[-\s]\d{4}",  # Credit card number pattern
+    r"\(?\d{3}\)?[-\s.]?\d{3}[-\s.]?\d{4}",  # Phone number
+    r"(0[1-9]|1[0-2])[-/.](0[1-9]|[12][0-9]|3[01])[-/.](19|20)\d\d",  # date of birth
+    r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)",  # IP address
+    r"[a-zA-Z0-9]{32}",  # API key
+    r"^(\d{5}.\d{2}-\d)|(\d{8})$"
+]
+
+ATTRIBUTE_PATTERNS = [
+    "documentNumber",
+    "documentCustodyAgentAccountCode",
+    "isinCode",
+    "payingAgentAccountCode",
+    "registrationParticipantAccountCode",
+    "nome",
+    "$ref",
+    "cpf",
+    "teste",
+    "valor",
+    "original",
+    "type",
+    "solicitacaoPagador",
+    "expiracao",
+    "chave",
+    "description",
+    "items",
+    "required",
+    "x-scope",
+    "maxLength"
+]
 
 #### IDCS Routines
 #### https://docs.oracle.com/en/learn/apigw-modeldeployment/index.html#introduction
@@ -51,20 +88,6 @@ def replace_regex(variavel):
 
     return variavel
 
-def replace_escape_chars(obj):
-    for k, v in obj.items():
-        if isinstance(v, str):
-            obj[k] = replace_regex(v)
-        elif isinstance(v, dict):
-            obj[k] = replace_escape_chars(v)
-        elif isinstance(v, list):
-            for i in range(len(v)):
-                if isinstance(v[i], str):
-                    v[i] = replace_regex(v[i])
-                elif isinstance(v[i], dict):
-                    v[i] = replace_escape_chars(v[i])
-    return obj
-
 def remove_property(dictionary, property_name):
     keys_to_delete = [key for key in dictionary if key == property_name]
     for key in keys_to_delete:
@@ -82,6 +105,14 @@ def remove_property(dictionary, property_name):
                     remove_property(item, property_name)
     return dictionary
 
+def count_attributes(json_data):
+    count = 0
+    for key, value in json_data.items():
+        count += 1
+        if isinstance(value, dict):
+            count += count_attributes(value)
+    return count
+
 def handler(ctx, data: io.BytesIO = None):
     config = oci.config.from_file("config")
     logging = oci.loggingingestion.LoggingClient(config)
@@ -99,6 +130,25 @@ def handler(ctx, data: io.BytesIO = None):
         ClientId = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
         ClientSecret = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
+        # JSON Items counter
+        jsonData = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"]
+        jsonData = dict(json.loads(jsonData))
+        c = count_attributes(jsonData)
+        if (c > 21):
+            rdata = json.dumps({
+                "active": False,
+                "context": {
+                    "status_code": 401,
+                    "message": "JSON exception",
+                    "error": "JSON exception",
+                }})
+
+            return response.Response(
+                ctx,
+                status_code=401,
+                response_data=rdata
+            )
+
         try:
             body = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"]
             body = json.loads(body)
@@ -175,6 +225,8 @@ def handler(ctx, data: io.BytesIO = None):
                     )
                 except(Exception) as ex2:
                     error_msg = beautify_str(str(ex2))
+                    redaction = Redaction.Redaction()
+                    error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg)
                     put_logs_response = logging.put_logs(
                         log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan",
                         put_logs_details=oci.loggingingestion.models.PutLogsDetails(
@@ -216,15 +268,20 @@ def handler(ctx, data: io.BytesIO = None):
                     api_spec = apigateway_client.get_api_content(contents[1])
                     spec_dict = json.loads(api_spec.data.content)
                     spec_dict = remove_property(spec_dict, "pattern")
+
                     spec = Spec.from_dict(spec_dict, config=bravado_config)
                     try:
                         schema = spec_dict["definitions"][contents[0]]
                     except:
                         schema = spec_dict["components"]["schemas"][contents[0]]
+
                     schema_without_pattern = remove_property(schema, "pattern")
+
                     validate_object(spec, schema_without_pattern, body)
                 except (Exception) as ex3:
                     error_msg = beautify_str(str(ex3))
+                    redaction = Redaction.Redaction()
+                    error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg)
                     put_logs_response = logging.put_logs(
                         log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan",
                         put_logs_details=oci.loggingingestion.models.PutLogsDetails(