GPC Basic Example

This code-based example demonstrates the use of variables, adding tests, and general configuration.

Note

Container images are built with the DataKitchen Interface Layer and require specific files and file structures to be present in a container node. For more information, see GPC File Structure and Configuration.

File contents

config.json

The config.json file defines a Python script and a shell script to execute. (The script names must include file extensions, as in .py or .sh.)

{
    "apt-dependencies": [ ],
    "dependencies": [ ],
    "keys": {
        "python_script": {
            "script": "basic.py",
            "environment": {
                "SIMPLE_ENV_VAR": "simple_env_var",
                "JINJA_ENV_VAR": "{{basic_example_node.RECIPE_VAR}}",
                "VAULT_ENV_VAR": "#{vault://vault/url}"
            },
            "parameters": {
                "SIMPLE_PARAM": "simple_param",
                "JINJA_PARAM": "{{basic_example_node.RECIPE_VAR}}",
                "VAULT_PARAM": "#{vault://vault/url}"
            },
            "export": [
                "success"
            ]
        },
        "shell_script": {
            "script": "basic.sh",
            "environment": {
                "SIMPLE_ENV_VAR": "simple_env_var",
                "JINJA_ENV_VAR": "{{basic_example_node.RECIPE_VAR}}",
                "VAULT_ENV_VAR": "#{vault://vault/url}"
            }
        }
    }
}

basic.py

The basic.py script defines "success" as a global variable for export that will be used in tests. It then checks that each environment variable and parameter you want to inject exists in the Python dictionary or globals array and are defined. Finally, the script logs the values injected using Jinja expressions.

import os
import sys
import traceback

global success


# Validate environment variables
if 'SIMPLE_ENV_VAR' not in os.environ or not os.environ['SIMPLE_ENV_VAR']:
    LOGGER.error("Undefined SIMPLE_ENV_VAR")
    sys.exit(1)

if 'JINJA_ENV_VAR' not in os.environ or not os.environ['JINJA_ENV_VAR']:
    LOGGER.error("Undefined JINJA_ENV_VAR")
    sys.exit(1)

if 'VAULT_ENV_VAR' not in os.environ or not os.environ['VAULT_ENV_VAR']:
    LOGGER.error("Undefined VAULT_ENV_VAR")
    sys.exit(1)

# Validate parameters
if 'SIMPLE_PARAM' not in globals() or not SIMPLE_PARAM:
    LOGGER.error("Undefined SIMPLE_PARAM")
    sys.exit(1)

if 'JINJA_PARAM' not in globals() or not JINJA_PARAM:
    LOGGER.error("Undefined JINJA_PARAM")
    sys.exit(1)

if 'VAULT_PARAM' not in globals() or not VAULT_PARAM:
    LOGGER.error("Undefined VAULT_PARAM")
    sys.exit(1)

try:
    LOGGER.info(f'SIMPLE_ENV_VAR: {os.environ["SIMPLE_ENV_VAR"]}')
    LOGGER.info(f'JINJA_ENV_VAR: {os.environ["JINJA_ENV_VAR"]}')
    LOGGER.info(f'VAULT_ENV_VAR: {os.environ["VAULT_ENV_VAR"]}')
    LOGGER.info(f'SIMPLE_PARAM: {SIMPLE_PARAM}')
    LOGGER.info(f'JINJA_PARAM: {JINJA_PARAM}')
    LOGGER.info(f'VAULT_PARAM: {VAULT_PARAM}')
    LOGGER.info('EMBEDDED JINJA: {{basic_example_node.RECIPE_VAR}}')
    success = True
except Exception as e:
    LOGGER.error(f'Failed to read and log variables:
{traceback.format_exc()}')
    success = False

basic.sh

The basic.sh script checks that the environment variables are defined, then it prints the variable values. Finally, it writes a value to a file for use in testing.

#!/usr/bin/env bash

# Validate environment variables
if [ -z "$SIMPLE_ENV_VAR" ]; then
    echo "Undefined SIMPLE_ENV_VAR"
    exit 1
fi

if [ -z "$JINJA_ENV_VAR" ]; then
    echo "Undefined JINJA_ENV_VAR"
    exit 1
fi

if [ -z "$VAULT_ENV_VAR" ]; then
    echo "Undefined VAULT_ENV_VAR"
    exit 1
fi

# Print variables
echo "SIMPLE_ENV_VAR: $SIMPLE_ENV_VAR"
echo "JINJA_ENV_VAR: $JINJA_ENV_VAR"
echo "VAULT_ENV_VAR: $VAULT_ENV_VAR"
echo "EMBEDDED JINJA: {{basic_example_node.RECIPE_VAR}}

# Write to a file to demonstrate file export and testing
echo -n  "shell_script_output_value" > "docker-share/shell_script_output.txt"

#!/usr/bin/env bash

# Validate environment variables
if [ -z "$SIMPLE_ENV_VAR" ]; then
    echo "Undefined SIMPLE_ENV_VAR"
    exit 1
fi

if [ -z "$JINJA_ENV_VAR" ]; then
    echo "Undefined JINJA_ENV_VAR"
    exit 1
fi

if [ -z "$VAULT_ENV_VAR" ]; then
    echo "Undefined VAULT_ENV_VAR"
    exit 1
fi

# Print variables
echo "SIMPLE_ENV_VAR: $SIMPLE_ENV_VAR"
echo "JINJA_ENV_VAR: $JINJA_ENV_VAR"
echo "VAULT_ENV_VAR: $VAULT_ENV_VAR"
echo "EMBEDDED JINJA: {{basic_example_node.RECIPE_VAR}}"

# Write to a file to demonstrate file export and testing
echo -n  "shell_script_output_value" > "docker-share/shell_script_output.txt"

notebook.json

The notebook.json file holds the container configuration and test settings.

{
    "image-repo": "{{dockerhubConfig.image_repo.general_purpose}}",
    "image-tag": "{{dockerhubConfig.image_tag.general_purpose}}",
    "dockerhub-namespace": "{{dockerhubConfig.namespace.general_purpose}}",
    "dockerhub-username": "{{dockerhubConfig.username}}",
    "dockerhub-password": "{{dockerhubConfig.password}}",
    "assign-variables": [
        {
            "name": "shell_script_output",
            "file": "shell_script_output.txt"
        }
    ],
    "tests": {
        "log_dockerhub_tool_instance": {
            "test-variable": "dockerhubConfig",
            "action": "log",
            "type": "test-contents-as-string",
            "test-logic": "dockerhubConfig",
            "keep-history": true,
            "description": "Logs the DockerHub tool instance."
        },
        "test_success": {
            "test-variable": "success",
            "action": "stop-on-error",
            "type": "test-contents-as-boolean",
            "test-logic": "success",
            "keep-history": true,
            "description": "Stops the OrderRun if success is False."
        },
        "test_shell_script_output": {
            "test-variable": "shell_script_output",
            "action": "stop-on-error",
            "type": "test-contents-as-string",
            "test-logic": "shell_script_output == 'shell_script_output_value'",
            "keep-history": true,
            "description": "Stops the OrderRun if the shell script output is unexpected."
        }
    }
}