From 83b2702939435f2bf438a1cbf63636a4839f805c Mon Sep 17 00:00:00 2001 From: kev Date: Fri, 23 Feb 2024 17:16:21 +0800 Subject: [PATCH] update memgraph --- memgraph/example/dnslookup | 36 +++++++++++++++++++++++++++++++++++ memgraph/example/dump-elk.sh | 16 ++++++++++++++++ memgraph/example/load-csv.cql | 27 ++++++++++++++++++++++++++ memgraph/example/load-csv.sh | 15 +++++++++++++++ 4 files changed, 94 insertions(+) create mode 100755 memgraph/example/dnslookup create mode 100755 memgraph/example/dump-elk.sh create mode 100644 memgraph/example/load-csv.cql create mode 100755 memgraph/example/load-csv.sh diff --git a/memgraph/example/dnslookup b/memgraph/example/dnslookup new file mode 100755 index 0000000..a72714b --- /dev/null +++ b/memgraph/example/dnslookup @@ -0,0 +1,36 @@ +#!/bin/bash + +HOST=${1:?DOMAIN or IP is empty} + +docker exec -i memgraph_memgraph_1 mgconsole -output-format=csv <<- _CQL_ | sed -e 's/"//g' | tail -n +2 | tr '[],[]' ' ' | shuf | gawk -f /dev/fd/3 3<<- "_AWK_" | column -t -i1 -p2 -r3 -H1,2 | sed 's/─/& /' + MATCH p=(n)-[*]->(m) + WHERE any(n in nodes(p) where n.name = '$HOST') AND not exists(()-->(n)) AND not exists((m)-->()) + UNWIND nodes(p) AS nn + WITH DISTINCT nn + CALL path.expand(nn,[">"],[],1,1) YIELD result + RETURN extract(i in nodes(result)|i.name); +_CQL_ + BEGIN { + split("", cache); + split("", roots); + idx=0; + } + { + if(!($1 in cache)) { + roots[$1] = cache[$1] = ++idx; + } + if(!($2 in cache)) { + cache[$2] = ++idx; + } + delete roots[$2]; + print cache[$2], cache[$1], $2; + } + END { + print "0 -1 ." + for(root in roots) { + print cache[root], 0, root; + } + } +_AWK_ + +# vim: set noai noet: diff --git a/memgraph/example/dump-elk.sh b/memgraph/example/dump-elk.sh new file mode 100755 index 0000000..9201d40 --- /dev/null +++ b/memgraph/example/dump-elk.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +export PATH=/usr/local/bin:$PATH + +date=${1:-$(date -d yesterday +%F)} +url=http://127.0.0.1:9200/ +index=logstash-${date//-/.} +output=data/${date}.csv + +mkdir -p ${output%/*} + +elastic-query-export -c $url -i $index -o $output -q '+project:dns -_exists_:message +type:(A CNAME)' -fields '@timestamp,region,client,server,type,query,answer,ttl' + +gzip $output diff --git a/memgraph/example/load-csv.cql b/memgraph/example/load-csv.cql new file mode 100644 index 0000000..7476a6c --- /dev/null +++ b/memgraph/example/load-csv.cql @@ -0,0 +1,27 @@ +LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row +WITH DISTINCT row.query AS query +MERGE (d:Domain {name: query}); + +LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row +WITH DISTINCT row.answer AS answer WHERE row.type = 'CNAME' +MERGE (d:Domain {name: answer}); + +LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row +WITH DISTINCT row.answer AS answer WHERE row.type = 'A' +MERGE (i:IPv4 {name: answer}); + +LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row +WITH row WHERE row.type = 'A' +MATCH (d:Domain {name: row.query}), (i:IPv4 {name: row.answer}) +MERGE (d)-[r:A]->(i) +ON CREATE SET r.created_at = timestamp()/1000000 +CALL date.parse(replace(row.`@timestamp`, "Z", ""), "s", "%Y-%m-%dT%H:%M:%S.%f", "UTC") YIELD parsed +SET r.updated_at = parsed; + +LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row +WITH row WHERE row.type = 'CNAME' +MATCH (d1:Domain {name: row.query}), (d2:Domain {name: row.answer}) +MERGE (d1)-[r:CNAME]->(d2) +ON CREATE SET r.created_at = timestamp()/1000000 +CALL date.parse(replace(row.`@timestamp`, "Z", ""), "s", "%Y-%m-%dT%H:%M:%S.%f", "UTC") YIELD parsed +SET r.updated_at = parsed; diff --git a/memgraph/example/load-csv.sh b/memgraph/example/load-csv.sh new file mode 100755 index 0000000..38a591a --- /dev/null +++ b/memgraph/example/load-csv.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +FILE=$(date -d yesterday +%F).csv.gz +SRC_FILE=/data/dns/data/$FILE +DST_FILE=/path/to/dns.csv.gz + +echo "$(date +%FT%T) GET $FILE" +scp elk-us:$SRC_FILE $DST_FILE + +echo "$(date +%FT%T) LOAD CSV" +docker exec -i memgraph_memgraph_1 mgconsole < load-csv.cql + +echo "$(date +%FT%T) DONE"