Container Node Examples

Container node input file keys

Use this expression to inject files from a data source inside a container. Where:

• The key field expresses a reference to a key in a data source.
• The file name is a name relative to the folder defined by the field inside-container-file-directory. The docker-share folder is the default.

notebook.json excerpt

"container-input-file-keys": [
  {
    "key"       : "[ data source name ].[ key name ]",
    "filename"  : "some_input_file.csv"
  }
]

List of input files with wildcards

The * in the key field will be replaced by each key in the datasource, and the one in filename by the key name. See File-Based Wildcards for more details about how to retrieve multiple files using wildcards in datasources.

notebook.json

"container-input-file-keys": [
  {
    "key"       : "inputfiles.*",
    "filename"  : "*.csv"
  }
]

Container node output file keys

Use this expression to export files from inside the container to a data sink.

notebook.json excerpt

"container-output-file-keys" : [
  {
    "key"       : "outputfiles.some-output",
    "filename"  : "results.xml"
  }
]

List of output files with wildcards

Use the * in the filename field to export multiple files or files without specific names.

notebook.json

"container-output-file-keys" : [
  {
    "filename"  : "*.xml"
    "key"       : "store-results.*",
  }
]

Runtime output variables

It is possible to feed runtime variables with the contents of files produced by the container.

These variables are available for tests and further usage in the following nodes in the graph. Optionally, files can be decoded as JSON, which by default is read as plain text.

notebook.json

"assign-variables" : {
    "name": "variablename",
    "file": "output.json",
    "decode-json": true
}

Python3 with vault secrets

description.json

{
    "type": "DKNode_Container",
    "description": "Runs python3 with secrets from global and kitchen vault."
}

 

notebook.json

{
    "dockerhub-username": "#{vault://dockerhub/username}",
    "dockerhub-namespace": "datakitchen",
    "dockerhub-password": "#{vault://dockerhub/password}",
    "image-repo": "ac_python3_public_container",
    "metadata": {
        "name": "python3_container"
    },
    "analytic-container": true,
    "tests": {
        "test_global_key": {
            "test-logic": "result_global == 'global_val'",
            "action": "stop-on-error",
            "type": "test-contents-as-string",
            "test-variable": "result_global",
            "keep-history": false
        },
        "test_kitchen_key": {
            "test-logic": "result_kitchen == 'kitchen_value'",
            "action": "stop-on-error",
            "type": "test-contents-as-string",
            "test-variable": "result_kitchen",
            "keep-history": false
        }
    }
}

 

/docker-share/config.json

{
    "dependencies": [],
    "keys": {
        "run-script": {
            "script": "test.py",
            "parameters": {
                "global_secret": "#{vault://global/key}",
                "kitchen_secret": "#{vault://kitchen/key}"
            },
            "export": [
                "result_global",
                "result_kitchen"
            ]
        }
    }
}

 

/docker-share/test.py

import os

if __name__ == '__main__':
    global result_global, result_kitchen

    LOGGER.info("global secret: " + global_secret)
    LOGGER.info("kitchen secret: " + kitchen_secret)

    result_global = global_secret
    result_kitchen = kitchen_secret

ac_python3_container2 image

description.json

{
    "type": "DKNode_Container",
    "description": "Runs the ac_python3_container2 image."
}

 

notebook.json

{
    "dockerhub-username": "#{vault://dockerhub/username}",
    "dockerhub-namespace": "datakitchen",
    "dockerhub-password": "#{vault://dockerhub/password}",
    "image-repo": "ac_python3_container2",
    "metadata": {
        "name": "python3_container"
    },
    "analytic-container": true,
    "tests": {
        "test-filecount": {
            "test-logic": {
                "test-compare": "equal-to",
                "test-metric": 10
            },
            "action": "stop-on-error",
            "type": "test-contents-as-integer",
            "test-variable": "result",
            "keep-history": false
        },
        "test-float": {
            "test-logic": {
                "test-compare": "equal-to",
                "test-metric": 1.234
            },
            "action": "stop-on-error",
            "type": "test-contents-as-float",
            "test-variable": "float_val",
            "keep-history": false
        }
    },
    "assign-variables": [
        {
            "name": "float_val",
            "file": "float.txt"
        }
    ]
}

 

/docker-share/config.json

{
    "dependencies": [],
    "keys": {
        "run-script": {
            "script": "test.py",
            "parameters": {
                "globalvar1": "value1",
                "dockerhub_username": "{{dockerhub_username}}"
            },
            "export": [
                "result"
            ]
        }
    }
}

 

/docker-share/float.txt

1.234

 

/docker-share/records.csv

row1
row2
row3
row4
row5
row6
row7
row8
row9
row10

 

/docker-share/test.py

import os

if __name__ == '__main__':
    global result

    LOGGER.info('Value of globalvar1:' + globalvar1)
    LOGGER.info("(should work) dockerhub username: " + dockerhub_username)
    LOGGER.info("(should not work) dockerhub password: " + "{{dockerhub_password}}")


    with open('docker-share/records.csv') as f:
        result = len(f.readlines())