7c84501d-d10c-407c-b9f3-1d80e38fe36a
Nifi_and_Schema_Registry_Integration_End_to_End_Example-Single-Source-Topic-v4
9537bde7-2dca-37ec-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
nifi-record-serialization-services-nar
org.apache.nifi
1.5.0.3.1.0.0-501
Schema Write Strategy
Schema Write Strategy
schema-access-strategy
schema-access-strategy
schema-registry
org.apache.nifi.schemaregistry.services.SchemaRegistry
schema-registry
schema-name
schema-name
schema-text
schema-text
Date Format
Date Format
Time Format
Time Format
Timestamp Format
Timestamp Format
CSV Format
CSV Format
Value Separator
Value Separator
Include Header Line
Include Header Line
Quote Character
Quote Character
Escape Character
Escape Character
Comment Marker
Comment Marker
Null String
Null String
Trim Fields
Trim Fields
Quote Mode
Quote Mode
Record Separator
Record Separator
Include Trailing Delimiter
Include Trailing Delimiter
csvutils-character-set
csvutils-character-set
CSV Writer - HWX SR Registry
false
Schema Write Strategy
schema-access-strategy
schema-name
schema-registry
ae3a8974-0fa9-3a43-0000-000000000000
schema-name
schema-text
Date Format
Time Format
Timestamp Format
CSV Format
Value Separator
|
Include Header Line
false
Quote Character
Escape Character
Comment Marker
Null String
Trim Fields
Quote Mode
Record Separator
Include Trailing Delimiter
true
csvutils-character-set
DISABLED
org.apache.nifi.csv.CSVRecordSetWriter
ae3a8974-0fa9-3a43-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
nifi-hwx-schema-registry-nar
org.apache.nifi
1.5.0.3.1.0.0-501
url
url
cache-size
cache-size
cache-expiration
cache-expiration
HWX Schema Registry
false
url
http://hdf-hoosac3.field.hortonworks.com:7788/api/v1
cache-size
1000
cache-expiration
1 hour
DISABLED
org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry
afa5c551-475f-379b-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
nifi-record-serialization-services-nar
org.apache.nifi
1.5.0.3.1.0.0-501
schema-access-strategy
schema-access-strategy
schema-registry
org.apache.nifi.schemaregistry.services.SchemaRegistry
schema-registry
schema-name
schema-name
schema-text
schema-text
csv-reader-csv-parser
csv-reader-csv-parser
Date Format
Date Format
Time Format
Time Format
Timestamp Format
Timestamp Format
CSV Format
CSV Format
Value Separator
Value Separator
Skip Header Line
Skip Header Line
ignore-csv-header
ignore-csv-header
Quote Character
Quote Character
Escape Character
Escape Character
Comment Marker
Comment Marker
Null String
Null String
Trim Fields
Trim Fields
csvutils-character-set
csvutils-character-set
CSV Reader - HWX SR Registry
false
schema-access-strategy
schema-name
schema-registry
ae3a8974-0fa9-3a43-0000-000000000000
schema-name
schema-text
csv-reader-csv-parser
Date Format
Time Format
Timestamp Format
CSV Format
Value Separator
|
Skip Header Line
ignore-csv-header
Quote Character
Escape Character
Comment Marker
Null String
Trim Fields
csvutils-character-set
DISABLED
org.apache.nifi.csv.CSVReader
228551f0-1b17-343d-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
nifi-scripting-nar
org.apache.nifi
1.5.0.3.1.0.0-501
Script Engine
Script Engine
Script File
Script File
Script Body
Script Body
Module Directory
Module Directory
googleApiKey
googleApiKey
Enrich- ReverseGeoCodeLookupService
false
Script Engine
Groovy
Script File
Script Body
import org.apache.nifi.controller.ControllerServiceInitializationContext
import org.apache.nifi.reporting.InitializationException
class GroovyLookupService implements LookupService<String> {
def apiKey
def log
@Override
Optional<String> lookup(Map<String,String> coords) {
def lat = coords['lat']
def lng = coords['lng']
def fmt_addr = (new groovy.json.JsonSlurper()).parseText(
new URL("https://maps.googleapis.com/maps/api/geocode/json?latlng=$lat,$lng&key=${apiKey}").text)
.results[0]?.formatted_address
Optional.ofNullable(fmt_addr?.toString())
}
@Override
Class<?> getValueType() {
return String
}
@Override
Set<String> getRequiredKeys() {
return ['lat','lng'] as Set<String>
}
void setLogger(logger) {
log = logger
}
void onEnabled(context) {
def googleApiKeyPropertyDescriptor = context.properties.find {k,v -> k.name == 'googleApiKey'}?.key
apiKey = context.getProperty(googleApiKeyPropertyDescriptor)?.evaluateAttributeExpressions()?.value
}
@Override
void initialize(ControllerServiceInitializationContext context) throws InitializationException {
}
@Override
Collection<ValidationResult> validate(ValidationContext context) {
return null
}
@Override
PropertyDescriptor getPropertyDescriptor(String name) {
return null
}
@Override
void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
}
@Override
List<PropertyDescriptor> getPropertyDescriptors() {
return null
}
@Override
String getIdentifier() {
return null
}
}
lookupService = new GroovyLookupService()
Module Directory
googleApiKey
AIzaSyA-h19EtKZMbh33QXCWVdaK5gCFfbS-GAk
DISABLED
org.apache.nifi.lookup.script.ScriptedLookupService
5f06fa5d-13be-3397-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
nifi-record-serialization-services-nar
org.apache.nifi
1.5.0.3.1.0.0-501
Schema Write Strategy
Schema Write Strategy
schema-access-strategy
schema-access-strategy
schema-registry
org.apache.nifi.schemaregistry.services.SchemaRegistry
schema-registry
schema-name
schema-name
schema-text
schema-text
compression-format
compression-format
Avro Writer - HWX SR Registry - HWX Content Encode
false
Schema Write Strategy
hwx-content-encoded-schema
schema-access-strategy
schema-name
schema-registry
ae3a8974-0fa9-3a43-0000-000000000000
schema-name
${schema.name}
schema-text
${avro.schema}
compression-format
DISABLED
org.apache.nifi.avro.AvroRecordSetWriter
b23b6e1a-1322-393f-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
0.0
0.0
424.1869812011719
535.7506103515625
25576191-9d2e-3ec1-0000-000000000000
46f95fe5-3024-31d8-0000-000000000000
88.1064677249376
455.0239382346426
25576191-9d2e-3ec1-9c3d-3379c2b0fe3b
bb7e30bc-7914-3fff-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
1 GB
10000
25576191-9d2e-3ec1-0000-000000000000
b7bd657a-5efd-3127-0000-000000000000
PROCESSOR
0 sec
1
success
0
0f18120b-03e1-37a4-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
0f18120b-03e1-37a4-9845-a075ed46f983
1 GB
10000
25576191-9d2e-3ec1-0000-000000000000
b9dbb466-7adf-3181-0000-000000000000
PROCESSOR
b9dbb466-7adf-3181-9ce5-9e282359177d
0 sec
1
0
14cdc0c3-1be0-3de0-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
1 GB
10000
25576191-9d2e-3ec1-0000-000000000000
b7bd657a-5efd-3127-0000-000000000000
PROCESSOR
0 sec
1
success
0
20562d55-428b-3486-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
20562d55-428b-3486-bc56-2f06f16f2bdb
1 GB
10000
25576191-9d2e-3ec1-0000-000000000000
e03a7f85-a1ae-3b15-0000-000000000000
PROCESSOR
e03a7f85-a1ae-3b15-b1ad-0e4f2dc5a63e
0 sec
1
success
0
5b4f0b4b-1ea9-3cd9-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
5b4f0b4b-1ea9-3cd9-8f4f-f04bbf8b6a0e
1 GB
10000
25576191-9d2e-3ec1-0000-000000000000
e2cf5a07-7cc4-3ed9-0000-000000000000
PROCESSOR
e2cf5a07-7cc4-3ed9-9785-3f0d159b2879
0 sec
1
0
005011fa-d84a-369e-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
291.7634403323747
592.1604407684256
60.720367431640625
332.85711669921875
2990cf33-f5ff-3b2e-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
742.4869915509632
1146.3572609855723
2990cf33-f5ff-3b2e-819e-2990ebe6cf68
86.7586898803711
670.8236083984375
3b8d7fac-e922-3ed2-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
424.71635170932785
794.9564612762381
3b8d7fac-e922-3ed2-9377-76ec209def8d
44.64159393310547
196.63572692871094
5651d8eb-af49-3e0c-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
713.4010606937682
297.1606073398341
5651d8eb-af49-3e0c-8f0e-cfb043daca55
57.396385192871094
221.0823516845703
769e4b1c-276f-3f25-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
951.6170873274109
272.8449712767856
769e4b1c-276f-3f25-b217-693c19542b2a
bb764024-38f0-3a95-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
bb764024-38f0-3a95-9e99-a1ba1b6e77a0
1 GB
10000
769e4b1c-276f-3f25-0000-000000000000
1537ad38-7362-3033-0000-000000000000
OUTPUT_PORT
1537ad38-7362-3033-8d06-7ba9d07f60fe
0 sec
1
truck_speed_event
0
dd6214f4-fd45-377b-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
dd6214f4-fd45-377b-b619-2350bb1d6f22
1 GB
10000
769e4b1c-276f-3f25-0000-000000000000
b409f6c7-d8d6-3872-0000-000000000000
OUTPUT_PORT
b409f6c7-d8d6-3872-8237-ffeae25d3936
0 sec
1
truck_geo_event
0
52f10b4c-041b-3d55-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
1 GB
10000
769e4b1c-276f-3f25-0000-000000000000
c9f0bbd3-e5aa-3a23-0000-000000000000
PROCESSOR
c9f0bbd3-e5aa-3a23-bba2-dda3f67d3366
0 sec
1
success
0
ad49db3a-2250-30ba-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
843.08687290535
396.12503189857784
88.87496948242188
407.41314697265625
1989d676-45c7-3838-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
270.28120884285
161.5592970352966
1989d676-45c7-3838-9bd1-16aa1843adbb
194.44070434570312
916.2188110351562
792ffedb-1b6e-31c4-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
863.08687290535
622.1250318985778
792ffedb-1b6e-31c4-9c94-5745bf0acbcf
46.433006286621094
373.6318359375
b409f6c7-d8d6-3872-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
293.58835629321584
891.8865975103688
b409f6c7-d8d6-3872-8237-ffeae25d3936
1
Truck Geo Events
STOPPED
OUTPUT_PORT
1537ad38-7362-3033-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
805.0629656682158
899.0211189947438
1537ad38-7362-3033-8d06-7ba9d07f60fe
1
Truck Speed Events
STOPPED
OUTPUT_PORT
c9f0bbd3-e5aa-3a23-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
468.0038717554927
593.4203950037886
c9f0bbd3-e5aa-3a23-bba2-dda3f67d3366
nifi-standard-nar
org.apache.nifi
1.5.0.3.1.0.0-501
WARN
1
record-reader
org.apache.nifi.serialization.RecordReaderFactory
record-reader
record-writer
org.apache.nifi.serialization.RecordSetWriterFactory
record-writer
include-zero-record-flowfiles
include-zero-record-flowfiles
cache-schema
cache-schema
truck_geo_event
truck_geo_event
truck_speed_event
truck_speed_event
ALL
false
30 sec
record-reader
afa5c551-475f-379b-0000-000000000000
record-writer
9537bde7-2dca-37ec-0000-000000000000
include-zero-record-flowfiles
false
cache-schema
false
truck_geo_event
SELECT * FROM FLOWFILE where eventSource = 'truck_geo_event'
truck_speed_event
SELECT * FROM FLOWFILE where eventSource = 'truck_speed_event'
0
0 sec
TIMER_DRIVEN
1 sec
Route Telemetry Events
true
failure
true
original
false
truck_geo_event
false
truck_speed_event
STOPPED
org.apache.nifi.processors.standard.QueryRecord
0b1e9a53-c292-3af8-0000-000000000000
769e4b1c-276f-3f25-0000-000000000000
476.00000000000205
376.9999984528406
nifi-kafka-1-0-nar
org.apache.nifi
1.5.0.3.1.0.0-501
WARN
1
bootstrap.servers
bootstrap.servers
topic
topic
topic_type
topic_type
record-reader
org.apache.nifi.serialization.RecordReaderFactory
record-reader
record-writer
org.apache.nifi.serialization.RecordSetWriterFactory
record-writer
honor-transactions
honor-transactions
security.protocol
security.protocol
sasl.kerberos.service.name
sasl.kerberos.service.name
sasl.kerberos.principal
sasl.kerberos.principal
sasl.kerberos.keytab
sasl.kerberos.keytab
ssl.context.service
org.apache.nifi.ssl.SSLContextService
ssl.context.service
group.id
group.id
auto.offset.reset
auto.offset.reset
message-header-encoding
message-header-encoding
header-name-regex
header-name-regex
max.poll.records
max.poll.records
max-uncommit-offset-wait
max-uncommit-offset-wait
ALL
false
30 sec
bootstrap.servers
hdf-hoosac4.field.hortonworks.com:6667
topic
raw-all_truck_events_csv
topic_type
names
record-reader
afa5c551-475f-379b-0000-000000000000
record-writer
9537bde7-2dca-37ec-0000-000000000000
honor-transactions
true
security.protocol
PLAINTEXT
sasl.kerberos.service.name
sasl.kerberos.principal
sasl.kerberos.keytab
ssl.context.service
group.id
2
auto.offset.reset
latest
message-header-encoding
UTF-8
header-name-regex
schema.*
max.poll.records
10000
max-uncommit-offset-wait
1 secs
0
0 sec
TIMER_DRIVEN
1 sec
ConsumeKafkaRecord_1_0
true
parse.failure
false
success
STOPPED
org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_1_0
Acquire Events
b7bd657a-5efd-3127-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
971.6871708080607
977.3827511925354
nifi-kafka-1-0-nar
org.apache.nifi
1.5.0.3.1.0.0-501
WARN
1
bootstrap.servers
bootstrap.servers
topic
topic
record-reader
org.apache.nifi.serialization.RecordReaderFactory
record-reader
record-writer
org.apache.nifi.serialization.RecordSetWriterFactory
record-writer
use-transactions
use-transactions
acks
acks
attribute-name-regex
attribute-name-regex
message-header-encoding
message-header-encoding
security.protocol
security.protocol
sasl.kerberos.service.name
sasl.kerberos.service.name
sasl.kerberos.principal
sasl.kerberos.principal
sasl.kerberos.keytab
sasl.kerberos.keytab
ssl.context.service
org.apache.nifi.ssl.SSLContextService
ssl.context.service
message-key-field
message-key-field
max.request.size
max.request.size
ack.wait.time
ack.wait.time
max.block.ms
max.block.ms
partitioner.class
partitioner.class
compression.type
compression.type
ALL
false
30 sec
bootstrap.servers
hdf-hoosac4.field.hortonworks.com:6667
topic
${kafka.topic}
record-reader
afa5c551-475f-379b-0000-000000000000
record-writer
5f06fa5d-13be-3397-0000-000000000000
use-transactions
true
acks
all
attribute-name-regex
message-header-encoding
UTF-8
security.protocol
PLAINTEXT
sasl.kerberos.service.name
sasl.kerberos.principal
sasl.kerberos.keytab
ssl.context.service
message-key-field
max.request.size
1 MB
ack.wait.time
5 secs
max.block.ms
5 sec
partitioner.class
org.apache.kafka.clients.producer.internals.DefaultPartitioner
compression.type
none
0
0 sec
TIMER_DRIVEN
1 sec
Publish Events to Truck Stream Topics
true
failure
true
success
STOPPED
org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_1_0
b9dbb466-7adf-3181-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
642.7752763630203
537.3976624518189
b9dbb466-7adf-3181-9ce5-9e282359177d
nifi-update-attribute-nar
org.apache.nifi
1.5.0.3.1.0.0-501
WARN
1
Delete Attributes Expression
Delete Attributes Expression
Store State
Store State
Stateful Variables Initial Value
Stateful Variables Initial Value
kafka.topic
kafka.topic
schema.name
schema.name
ALL
false
30 sec
Delete Attributes Expression
Store State
Do not store state
Stateful Variables Initial Value
kafka.topic
truck_events_avro
schema.name
truck_events_avro
0
0 sec
TIMER_DRIVEN
1 sec
Specify Schema and Destination
false
success
STOPPED
org.apache.nifi.processors.attributes.UpdateAttribute
e03a7f85-a1ae-3b15-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
639.2836500026511
747.9535972551678
e03a7f85-a1ae-3b15-b1ad-0e4f2dc5a63e
nifi-standard-nar
org.apache.nifi
1.5.0.3.1.0.0-501
WARN
1
record-reader
org.apache.nifi.serialization.RecordReaderFactory
record-reader
record-writer
org.apache.nifi.serialization.RecordSetWriterFactory
record-writer
lookup-service
org.apache.nifi.lookup.LookupService
lookup-service
result-record-path
result-record-path
routing-strategy
routing-strategy
result-contents
result-contents
lat
lat
lng
lng
ALL
false
30 sec
record-reader
afa5c551-475f-379b-0000-000000000000
record-writer
9537bde7-2dca-37ec-0000-000000000000
lookup-service
228551f0-1b17-343d-0000-000000000000
result-record-path
/geoAddress
routing-strategy
route-to-success
result-contents
insert-entire-record
lat
/latitude
lng
/longitude
0
0 sec
TIMER_DRIVEN
1 sec
ReverseGeoLookup
true
failure
false
success
STOPPED
org.apache.nifi.processors.standard.LookupRecord
e2cf5a07-7cc4-3ed9-0000-000000000000
25576191-9d2e-3ec1-0000-000000000000
1306.5787740461533
540.9518252866341
e2cf5a07-7cc4-3ed9-9785-3f0d159b2879
nifi-update-attribute-nar
org.apache.nifi
1.5.0.3.1.0.0-501
WARN
1
Delete Attributes Expression
Delete Attributes Expression
Store State
Store State
Stateful Variables Initial Value
Stateful Variables Initial Value
kafka.topic
kafka.topic
schema.name
schema.name
ALL
false
30 sec
Delete Attributes Expression
Store State
Do not store state
Stateful Variables Initial Value
kafka.topic
truck_speed_events_avro
schema.name
truck_speed_events_avro
0
0 sec
TIMER_DRIVEN
1 sec
Specify Schema and Destination
false
success
STOPPED
org.apache.nifi.processors.attributes.UpdateAttribute
Use Case 1
92282209-5b27-409c-8e2e-1c4416056448
HDF-Trucking-Reference-App
Trucking Reference App for Nifi for HDF 3.0
d032b8a1-3624-46fb-88b0-1a3e0ebd1003
Nifi_and_Schema_Registry_Integration_End_to_End_Example-Single-Source-Topic
0ea802d1-0161-1000-ffff-ffff94016385
Nifi Registry
LOCALLY_MODIFIED
Local changes have been made
1
01/22/2018 22:31:42 UTC