Using self-hosted credentials

📘
This feature is only supported in "Cloud," "Cloud with Customer-hosted ObjectStorage Deployments," and with the Data Collector (Deprecated) for AWS.
See all current deployment options here.

Monte Carlo provides the option to store integration credentials in your AWS account using Secrets Manager.

To configure, follow these steps -

Create a read-only service account or role for the integration.
See detailed instructions in our dedicated guides for Warehouses, Lakes and BI Tools.
Create a secret in your AWS account.
Create a role that allows secret access for Monte Carlo's data collector / collection service.
Set up the integration using your self-hosted credentials.

🚧
If your integration is already connected to MC, please see FAQs for how to migrate the connection to using self hosted credentials! You do not need to remake the integration.

Create a secret in your AWS account

You will create a secret containing the credentials for the integration's service account or role.

Select and fill in the credentials schema associated with the resource you wish to integrate with.

{
	"catalog": {
		"type": "string",
		"required": false
	},
	"assumable_role": {
		"type": "string",
		"required": false
	},
	"external_id": {
		"type": "string",
		"required": false
	},
	"workgroup": {
		"type": "string",
		"required": false
	},
	"aws_region": {
		"type": "string",
		"required": false
	}
}

{
	"type": {
		"type": "string",
		"required": true
	},
	"project_id": {
		"type": "string",
		"required": true
	},
	"private_key_id": {
		"type": "string",
		"required": true
	},
	"private_key": {
		"type": "string",
		"required": true
	},
	"client_email": {
		"type": "string",
		"required": true
	},
	"client_id": {
		"type": "string",
		"required": true
	},
	"auth_uri": {
		"type": "string",
		"required": true
	},
	"token_uri": {
		"type": "string",
		"required": true
	},
	"auth_provider_x509_cert_url": {
		"type": "string",
		"required": true
	},
	"client_x509_cert_url": {
		"type": "string",
		"required": true
	}
}

{
	"bucket": {
		"type": "string",
		"required": true
	},
	"prefix": {
		"type": "string",
		"required": true
	},
	"assumable_role": {
		"type": "string",
		"required": false
	},
	"external_id": {
		"type": "string",
		"required": false
	}
}

{
	"assumable_role": {
		"type": "string",
		"required": false
	},
	"external_id": {
		"type": "string",
		"required": false
	},
	"aws_region": {
		"type": "string",
		"required": false
	}
}

{
	"host": {
		"type": "string",
		"required": true
	},
	"port": {
		"type": "integer",
		"required": false,
		"nullable": false
	},
	"username": {
		"type": "string",
		"required": true
	},
	"database": {
		"type": "string",
		"required": false,
		"nullable": true
	},
	"password": {
		"type": "string",
		"required": true
	}
}

{
	"db_name": {
		"type": "string",
		"required": true
	},
	"host": {
		"type": "string",
		"required": true
	},
	"port": {
		"type": "string",
		"required": true
	},
	"user": {
		"type": "string",
		"required": true
	},
	"password": {
		"type": "string",
		"required": true
	},
	"assumable_role": {
		"type": "string",
		"required": false
	},
	"external_id": {
		"type": "string",
		"required": false
	},
	"ssl_options": {
		"type": "dict",
		"required": false,
		"schema": {
			"ca": {
				"type": "string",
				"required": false
			},
			"cert": {
				"type": "string",
				"required": false
			},
			"key": {
				"type": "string",
				"required": false
			},
			"mechanism": {
				"type": "string",
				"required": false,
				"allowed": ["url"]
			}
		}
	}
}

{
	"base_url": {
		"type": "string",
		"required": true
	},
	"client_id": {
		"type": "string",
		"required": true
	},
	"client_secret": {
		"type": "string",
		"required": true
	},
	"verify_ssl": {
		"type": "boolean",
		"required": false
	}
}

{
	"ssh_key": {
		"type": "string",
		"required": false
	},
	"repo_url": {
		"type": "string",
		"required": true
	},
	"token": {
		"type": "string",
		"required": false
	},
	"username": {
		"type": "string",
		"required": false
	}
}

{
	"host": {
		"type": "string",
		"required": true
	},
	"port": {
		"type": "integer",
		"required": false
	},
	"user": {
		"type": "string",
		"required": true
	},
	"catalog": {
		"type": "string",
		"required": false
	},
	"schema": {
		"type": "string",
		"required": false
	},
	"timeout": {
		"type": "integer",
		"required": false
	},
	"http_scheme": {
		"type": "string",
		"required": false
	},
	"password": {
		"type": "string",
		"required": false
	},
	"ssl_options": {
		"type": "dict",
		"required": false,
		"schema": {
			"ca": {
				"type": "string",
				"required": false
			},
			"cert": {
				"type": "string",
				"required": false
			},
			"key": {
				"type": "string",
				"required": false
			},
			"mechanism": {
				"type": "string",
				"required": false,
				"allowed": ["dc-s3"]
			},
			"skip_verification": {
				"type": "boolean",
				"required": false
			}
		}
	}
}

{
	"db_name": {
		"type": "string",
		"required": true
	},
	"host": {
		"type": "string",
		"required": true
	},
	"port": {
		"type": "string",
		"required": true
	},
	"user": {
		"type": "string",
		"required": true
	},
	"password": {
		"type": "string",
		"required": true
	}
}

{
	"user": {
		"type": "string",
		"required": true
	},
	"password": {
		"type": "string",
		"required": true
	},
	"account": {
		"type": "string",
		"required": true
	},
	"warehouse": {
		"type": "string",
		"required": false
	}
}

{
	"mode": "binary",
	"host": {
		"type": "string",
		"required": false
	},
	"port": {
		"type": "integer",
		"required": false,
		"nullable": true
	},
	"username": {
		"type": "string",
		"required": false
	},
	"database": {
		"type": "string",
		"required": false,
		"nullable": true
	},
	"password": {
		"type": "string",
		"required": false
	}
}

{
	"mode": "databricks",

	"databricks_workspace_url": {
		"type": "string",
		"required": false
	},
	"databricks_workspace_id": {
		"type": "string",
		"required": false
	},
	"databricks_cluster_id": {
		"type": "string",
		"required": false
	},
	"databricks_token": {
		"type": "string",
		"required": false
	}
}

{
	"mode": "http",
	"username": {
		"type": "string",
		"required": false
	},
	"password": {
		"type": "string",
		"required": false
	},
	"url": {
		"type": "string",
		"required": false
	}
}

{
    "server_name": {
        "type": "string",
        "required": true
    },
    "site_name": {
        "type": "string",
        "required": true
    },
    "token_name": {
        "type": "string",
        "required": true
    },
    "token_value": {
        "type": "string",
        "required": true
    }
}

Go to Secrets Manager on your AWS console and click "Store a new secret." Make sure to select the appropriate region to create your secret. The region should match that of your data collector. You can view your data collector's region by visiting your Monte Carlo settings page.
Select "Other type of secrets" and paste the filled-in schema from step 1. Then select "Next".

Give the secret a meaningful name, description and any tags. Then select "Next".
Configure automatic rotation, if you'd like. Then select "Next".
Review and select "Store".
From the Secrets Manager console search for the secret you just created. Save the ARN.

Create a role that allows secret access for Monte Carlo's data collector

You will create an IAM role with the necessary API permissions in order to access to the secret:

Copy the policy below. Please specify the ARN from above where <secret_arn> appears.

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": "secretsmanager:GetSecretValue",
            "Resource": "<secret_arn>"
        }
    ]
}

Follow the steps outlined here to create the IAM role. You will attach the policy from step 1 to this role as part of the process.

Set up the integration

You will provide connection details using Monte Carlo's CLI:

Follow this guide to install and configure the CLI (requires >= 0.13.2).
Use the command montecarlo integrations add-self-hosted-credentials to add the integration.
For reference, see help below:

$ montecarlo integrations add-self-hosted-credentials --help
Usage: montecarlo integrations add-self-hosted-credentials
           [OPTIONS]

  Setup an integration that uses self-hosted credentials.

Options:
  --connection-type [s3-airflow-log-events|hive-mysql|hive-s3|hive|presto|presto-s3|glue|athena|spark|redshift|snowflake|bigquery|looker|looker-git-ssh|looker-git-clone|dbt-cloud|tableau]
                                  Type of connection. This option requires
                                  setting 'name' when it is set to one of
                                  these values: {'bigquery', 'snowflake',
                                  'redshift'}.  [required]
  --mechanism [secretsmanager]    Credential self-hosting mechanism.
                                  [required]
  --key TEXT                      Identifier for credentials within self-
                                  hosting mechanism.  [required]
  --name TEXT                     Friendly name for the warehouse.
  --role TEXT                     Assumable role ARN to use for accessing AWS
                                  resources.
  --external-id TEXT              An external id, per assumable role
                                  conditions.
  --collector-id UUID             ID for the data collector. To disambiguate
                                  accounts with multiple collectors.
  --skip-validation               Skip all connection tests. This option
                                  cannot be used with 'validate-only'.
  --validate-only                 Run connection tests without adding. This
                                  option cannot be used with 'skip-
                                  validation'.
  --auto-yes                      Skip any interactive approval.
  --option-file FILE              Read configuration from FILE.
  --help                          Show this message and exit.

FAQs

What if I want to migrate an existing integration to use self-hosted credentials?
After creating the secret and role the command montecarlo integrations update can be used to transfer credential management.

Note - The command montecarlo integrations list can be used to echo the <CONNECTION-ID>.

$ montecarlo integrations update --replace-all --connection-id <CONNECTION-ID> --changes '{"self_hosting_mechanism": "secretsmanager", "self_hosting_key": "<SECRET_ARN>", "assumable_role": "<ROLE_ARN>", "external_id": "<EXTERNAL_ID>", "region": "<AWS_REGION>"}'