...
{
"type": "staging-connector",
"action": "lookup",
"name": "Extract additional data",
"sourceId": "/a/field/pointer",
"nestedSourceField": "/another/field/pointer",
"targetBucket": "a_bucket_name",
"connection": {
"servers": [
{
"host": "localhost",
"port": 8081
}
],
"connectTimeout": "PT5m",
"readTimeout": "PT5m"
}
}
Lookup action
This is configured as a processor and has the following fields:
...
{
"processAction": "process",
"seed": {
"servers": [{
"host": "localhost",
"port": 8081
}],
"connectTimeout": "PT5m",
"readTimeout": "PT5m",
"scan": {
"bucket": "source_bucket_name",
"scroll": {
"size": 100
},
"query": {
"filter": {
"or": [{
"exists": {
"fieldName": "content.metadata.Content-Type",
"present": false
}
},
{
"equals": {
"fieldName": "content.metadata.Content-Type",
"value": ""
}
}
]
}
}
}
},
"name": "Records missing content type",
"active": true,
"scanAction": "query",
"type": "staging-connector",
"batchSize": 25,
"properties": {
"index": "target_data_source_index_name"
},
"pipelineId": "{{ fromName('Persist to Target Data Source') }}",
"id": "db122b46-9104-48e7-8847-c1f2d720d596"
}
...