diff --git a/README.md b/README.md index 0d36976..b2c055b 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ First, you need to import the **bravado** library for **validate_object**. ![import-bravado.png](images/import-bravado.png) -This code is responsible to redact your validation response: +This code is responsible to beautify your validation response: ![redact-code.png](images/redact-code.png) @@ -174,10 +174,23 @@ And you can put in the main function code: ![main-redact-routine.png](images/main-redact-routine.png) -The response will be like this: +You need to use a library called [Redaction.py](./files/authApi/Redaction.py), included in this material. This class find your patterns for attributes that need redaction. -![redact-response.png](images/redact-response.png) +Declare your patterns and attribute names: +![redaction-2.png](images/redaction-2.png) + +So you can use the library like this: + +![redaction-1.png](images/redaction-1.png) + +When you request an REST service with this body content: + +![img.png](images/redaction-4.png) + +The response will be something like this: + +![redaction-3.png](images/redaction-3.png) ## Resource Principal diff --git a/files/OCI_API_Gateway_Automation_files.zip b/files/OCI_API_Gateway_Automation_files.zip index f85d557..4fe88eb 100644 Binary files a/files/OCI_API_Gateway_Automation_files.zip and b/files/OCI_API_Gateway_Automation_files.zip differ diff --git a/files/authApi/Redaction.py b/files/authApi/Redaction.py new file mode 100644 index 0000000..8df83bf --- /dev/null +++ b/files/authApi/Redaction.py @@ -0,0 +1,169 @@ +import re + +# This class does not work with numeric attributes, only with String +# Use the method redact and put your patterns as the example: +# SENSITIVE_PATTERNS = [ +# r"\d{3}-\d{2}-\d{4}", # Social Security Number (SSN) pattern +# r"\d{4}[-\s]\d{4}[-\s]\d{4}[-\s]\d{4}", # Credit card number pattern +# r"\(?\d{3}\)?[-\s.]?\d{3}[-\s.]?\d{4}", # Phone number +# r"(0[1-9]|1[0-2])[-/.](0[1-9]|[12][0-9]|3[01])[-/.](19|20)\d\d", # date of birth +# r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)", # IP address +# r"[a-zA-Z0-9]{32}" # API key +# ] +# +# ATTRIBUTE_PATTERNS = [ +# "nome", +# "cpf", +# "teste", +# "valor", +# "original", +# "type", +# "solicitacaoPagador", +# "chave", +# "description", +# "items", +# "example" +# ] + +class Redaction(): + + def repl_value2(self, message, pattern): + flag_aspas = False + flag_attribute = False + # flag_vezes = 0 + flag_dois_pontos = False + flag_colchetes = False + i = 0 + z = pattern + str_acc = "" + while (i < len(message)): + try: + if (message[i:i + len(z)] == z and (message[i + len(z):i + len(z) + 1] == "'" or message[i + len(z):i + len(z) + 1] == "\"")): + flag_attribute = True + flag_aspas = True + except: + print("except") + if (message[i] == ":" and not flag_aspas and flag_attribute): + flag_dois_pontos = True + if (flag_aspas and message[i] != "'" and message[i] != "\"" and flag_attribute and flag_dois_pontos): + str_acc = str_acc + message[i] + message = message[0:i] + "*" + message[i + 1:len(message) + i] + if (message[i] == "{" and flag_dois_pontos and not flag_aspas): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + if ((message[i] == "}" or message[i] == "]") and not flag_aspas): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + str_acc = "" + if (flag_dois_pontos and not flag_aspas and message[i] == "["): + flag_colchetes = True + if (message[i] == "," and not flag_aspas and not flag_colchetes): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + str_acc = "" + if ((message[i] == "'" or message[i] == "\"")): + flag_aspas = not flag_aspas + if (flag_aspas == False and flag_attribute == True and flag_dois_pontos and len(str_acc) > 0 and not flag_colchetes): + flag_attribute = False + flag_dois_pontos = False + str_acc = "" + i = i + 1 + return message + + def repl_value(self, message, pattern): + flag_aspas = False + flag_attribute = False + # flag_vezes = 0 + flag_dois_pontos = False + flag_colchetes = False + flag_string = True + flag_descobre_tipo = False + i = 0 + z = pattern + str_acc = "" + while (i < len(message)): + try: + if (message[i:i + len(z)] == z and (message[i + len(z):i + len(z) + 1] == "'" or message[i + len(z):i + len(z) + 1] == "\"")): + flag_attribute = True + flag_aspas = True + except: + print("except") + if (message[i] == ":" and not flag_aspas and flag_attribute): + flag_dois_pontos = True + flag_descobre_tipo = True + if ((flag_aspas and message[i] != "'" and message[i] != "\"" and flag_attribute and flag_dois_pontos) + or (message[i] in "0123456789." and flag_attribute and flag_dois_pontos and not flag_string)): + str_acc = str_acc + message[i] + message = message[0:i] + "*" + message[i + 1:len(message) + i] + if (message[i] == "{" and flag_dois_pontos and not flag_aspas): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + flag_descobre_tipo = False + flag_string = True + if ((message[i] == "}" or message[i] == "]") and not flag_aspas): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + flag_descobre_tipo = False + flag_string = True + str_acc = "" + if ((message[i] == "}" or message[i] == "]") and not flag_aspas and not flag_string): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + flag_descobre_tipo = False + flag_string = True + str_acc = "" + if (flag_dois_pontos and not flag_aspas and message[i] == "["): + flag_colchetes = True + if (message[i] == "," and not flag_aspas and not flag_colchetes and not flag_string): + flag_attribute = False + flag_dois_pontos = False + flag_aspas = False + flag_colchetes = False + flag_descobre_tipo = False + flag_string = True + str_acc = "" + if ((message[i] == "'" or message[i] == "\"")): + flag_aspas = not flag_aspas + if (flag_descobre_tipo): + flag_string = True + flag_descobre_tipo = False + if (message[i] in "01234567890." and flag_descobre_tipo): + flag_string = False + flag_descobre_tipo = False + str_acc = str_acc + message[i] + message = message[0:i] + "*" + message[i + 1:len(message) + i] + if (flag_aspas == False and flag_attribute == True and flag_dois_pontos and len(str_acc) > 0 and not flag_colchetes and flag_string): + flag_attribute = False + flag_dois_pontos = False + flag_descobre_tipo = False + str_acc = "" + i = i + 1 + return message + + def repl(self, attribute_pattern, message): + msg_return = [] + for pattern in attribute_pattern: + message = self.repl_value(message, pattern) + return message + + def change(self, sensitive_pattern, message): + for pattern in sensitive_pattern: + message = re.sub(pattern, "", message) + return message + + def redact(self, sensitive_pattern, attribute_pattern, message): + message = self.repl(attribute_pattern, message) + message = self.change(sensitive_pattern, message) + return message diff --git a/files/authApi/authRPApi.py b/files/authApi/authRPApi.py index bced7a6..7ef1fc4 100644 --- a/files/authApi/authRPApi.py +++ b/files/authApi/authRPApi.py @@ -10,6 +10,43 @@ import os import ast from bravado_core.spec import Spec from bravado_core.validate import validate_object +from datetime import datetime +from random import randrange + +import Redaction + +SENSITIVE_PATTERNS = [ + r"\d{3}-\d{2}-\d{4}", # Social Security Number (SSN) pattern + r"\d{4}[-\s]\d{4}[-\s]\d{4}[-\s]\d{4}", # Credit card number pattern + r"\(?\d{3}\)?[-\s.]?\d{3}[-\s.]?\d{4}", # Phone number + r"(0[1-9]|1[0-2])[-/.](0[1-9]|[12][0-9]|3[01])[-/.](19|20)\d\d", # date of birth + r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)", # IP address + r"[a-zA-Z0-9]{32}", # API key + r"^(\d{5}.\d{2}-\d)|(\d{8})$" +] + +ATTRIBUTE_PATTERNS = [ + "documentNumber", + "documentCustodyAgentAccountCode", + "isinCode", + "payingAgentAccountCode", + "registrationParticipantAccountCode", + "nome", + "$ref", + "cpf", + "teste", + "valor", + "original", + "type", + "solicitacaoPagador", + "expiracao", + "chave", + "description", + "items", + "required", + "x-scope", + "maxLength" +] #### IDCS Routines #### https://docs.oracle.com/en/learn/apigw-modeldeployment/index.html#introduction @@ -51,20 +88,6 @@ def replace_regex(variavel): return variavel -def replace_escape_chars(obj): - for k, v in obj.items(): - if isinstance(v, str): - obj[k] = replace_regex(v) - elif isinstance(v, dict): - obj[k] = replace_escape_chars(v) - elif isinstance(v, list): - for i in range(len(v)): - if isinstance(v[i], str): - v[i] = replace_regex(v[i]) - elif isinstance(v[i], dict): - v[i] = replace_escape_chars(v[i]) - return obj - def remove_property(dictionary, property_name): keys_to_delete = [key for key in dictionary if key == property_name] for key in keys_to_delete: @@ -82,6 +105,14 @@ def remove_property(dictionary, property_name): remove_property(item, property_name) return dictionary +def count_attributes(json_data): + count = 0 + for key, value in json_data.items(): + count += 1 + if isinstance(value, dict): + count += count_attributes(value) + return count + def read_secret_value(secret_client, secret_id): response = secret_client.get_secret_bundle(secret_id) base64_Secret_content = response.data.secret_bundle_content.content @@ -91,9 +122,8 @@ def read_secret_value(secret_client, secret_id): return secret_content def handler(ctx, data: io.BytesIO = None): - signer = oci.auth.signers.get_resource_principals_signer() - logging = oci.loggingingestion.LoggingClient(config={}, signer=signer) - secret_client = oci.secrets.SecretsClient(config={}, signer=signer) + config = oci.config.from_file("config") + logging = oci.loggingingestion.LoggingClient(config) # functions context variables app_context = dict(ctx.Config()) @@ -108,6 +138,25 @@ def handler(ctx, data: io.BytesIO = None): ClientId = read_secret_value(secret_client, "ocid1.vaultsecret.oc1.iad.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") ClientSecret = read_secret_value(secret_client, "ocid1.vaultsecret.oc1.iad.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") + # JSON Items counter + jsonData = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"] + jsonData = dict(json.loads(jsonData)) + c = count_attributes(jsonData) + if (c > 21): + rdata = json.dumps({ + "active": False, + "context": { + "status_code": 401, + "message": "JSON exception", + "error": "JSON exception", + }}) + + return response.Response( + ctx, + status_code=401, + response_data=rdata + ) + try: body = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"] body = json.loads(body) @@ -184,6 +233,8 @@ def handler(ctx, data: io.BytesIO = None): ) except(Exception) as ex2: error_msg = beautify_str(str(ex2)) + redaction = Redaction.Redaction() + error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg) put_logs_response = logging.put_logs( log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan", put_logs_details=oci.loggingingestion.models.PutLogsDetails( @@ -225,15 +276,20 @@ def handler(ctx, data: io.BytesIO = None): api_spec = apigateway_client.get_api_content(contents[1]) spec_dict = json.loads(api_spec.data.content) spec_dict = remove_property(spec_dict, "pattern") + spec = Spec.from_dict(spec_dict, config=bravado_config) try: schema = spec_dict["definitions"][contents[0]] except: schema = spec_dict["components"]["schemas"][contents[0]] + schema_without_pattern = remove_property(schema, "pattern") + validate_object(spec, schema_without_pattern, body) except (Exception) as ex3: error_msg = beautify_str(str(ex3)) + redaction = Redaction.Redaction() + error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg) put_logs_response = logging.put_logs( log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan", put_logs_details=oci.loggingingestion.models.PutLogsDetails( diff --git a/files/authApi/func.py b/files/authApi/func.py index 92aeb7b..cd44534 100644 --- a/files/authApi/func.py +++ b/files/authApi/func.py @@ -10,6 +10,43 @@ import os import ast from bravado_core.spec import Spec from bravado_core.validate import validate_object +from datetime import datetime +from random import randrange + +import Redaction + +SENSITIVE_PATTERNS = [ + r"\d{3}-\d{2}-\d{4}", # Social Security Number (SSN) pattern + r"\d{4}[-\s]\d{4}[-\s]\d{4}[-\s]\d{4}", # Credit card number pattern + r"\(?\d{3}\)?[-\s.]?\d{3}[-\s.]?\d{4}", # Phone number + r"(0[1-9]|1[0-2])[-/.](0[1-9]|[12][0-9]|3[01])[-/.](19|20)\d\d", # date of birth + r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)", # IP address + r"[a-zA-Z0-9]{32}", # API key + r"^(\d{5}.\d{2}-\d)|(\d{8})$" +] + +ATTRIBUTE_PATTERNS = [ + "documentNumber", + "documentCustodyAgentAccountCode", + "isinCode", + "payingAgentAccountCode", + "registrationParticipantAccountCode", + "nome", + "$ref", + "cpf", + "teste", + "valor", + "original", + "type", + "solicitacaoPagador", + "expiracao", + "chave", + "description", + "items", + "required", + "x-scope", + "maxLength" +] #### IDCS Routines #### https://docs.oracle.com/en/learn/apigw-modeldeployment/index.html#introduction @@ -51,20 +88,6 @@ def replace_regex(variavel): return variavel -def replace_escape_chars(obj): - for k, v in obj.items(): - if isinstance(v, str): - obj[k] = replace_regex(v) - elif isinstance(v, dict): - obj[k] = replace_escape_chars(v) - elif isinstance(v, list): - for i in range(len(v)): - if isinstance(v[i], str): - v[i] = replace_regex(v[i]) - elif isinstance(v[i], dict): - v[i] = replace_escape_chars(v[i]) - return obj - def remove_property(dictionary, property_name): keys_to_delete = [key for key in dictionary if key == property_name] for key in keys_to_delete: @@ -82,6 +105,14 @@ def remove_property(dictionary, property_name): remove_property(item, property_name) return dictionary +def count_attributes(json_data): + count = 0 + for key, value in json_data.items(): + count += 1 + if isinstance(value, dict): + count += count_attributes(value) + return count + def handler(ctx, data: io.BytesIO = None): config = oci.config.from_file("config") logging = oci.loggingingestion.LoggingClient(config) @@ -99,6 +130,25 @@ def handler(ctx, data: io.BytesIO = None): ClientId = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ClientSecret = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + # JSON Items counter + jsonData = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"] + jsonData = dict(json.loads(jsonData)) + c = count_attributes(jsonData) + if (c > 21): + rdata = json.dumps({ + "active": False, + "context": { + "status_code": 401, + "message": "JSON exception", + "error": "JSON exception", + }}) + + return response.Response( + ctx, + status_code=401, + response_data=rdata + ) + try: body = dict(json.loads(data.getvalue().decode('utf-8')).get("data"))["body"] body = json.loads(body) @@ -175,6 +225,8 @@ def handler(ctx, data: io.BytesIO = None): ) except(Exception) as ex2: error_msg = beautify_str(str(ex2)) + redaction = Redaction.Redaction() + error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg) put_logs_response = logging.put_logs( log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan", put_logs_details=oci.loggingingestion.models.PutLogsDetails( @@ -216,15 +268,20 @@ def handler(ctx, data: io.BytesIO = None): api_spec = apigateway_client.get_api_content(contents[1]) spec_dict = json.loads(api_spec.data.content) spec_dict = remove_property(spec_dict, "pattern") + spec = Spec.from_dict(spec_dict, config=bravado_config) try: schema = spec_dict["definitions"][contents[0]] except: schema = spec_dict["components"]["schemas"][contents[0]] + schema_without_pattern = remove_property(schema, "pattern") + validate_object(spec, schema_without_pattern, body) except (Exception) as ex3: error_msg = beautify_str(str(ex3)) + redaction = Redaction.Redaction() + error_msg = redaction.redact(sensitive_pattern=SENSITIVE_PATTERNS, attribute_pattern=ATTRIBUTE_PATTERNS, message=error_msg) put_logs_response = logging.put_logs( log_id="ocid1.log.oc1.iad.amaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaanamaaaaaan", put_logs_details=oci.loggingingestion.models.PutLogsDetails( diff --git a/images/redaction-1.png b/images/redaction-1.png new file mode 100644 index 0000000..ead5f9c Binary files /dev/null and b/images/redaction-1.png differ diff --git a/images/redaction-2.png b/images/redaction-2.png new file mode 100644 index 0000000..25fdc64 Binary files /dev/null and b/images/redaction-2.png differ diff --git a/images/redaction-3.png b/images/redaction-3.png new file mode 100644 index 0000000..1408d71 Binary files /dev/null and b/images/redaction-3.png differ diff --git a/images/redaction-4.png b/images/redaction-4.png new file mode 100644 index 0000000..4b890fa Binary files /dev/null and b/images/redaction-4.png differ