Load from Spark Dataframe

gsql '
    CREATE VERTEX Comment (PRIMARY_ID id UINT, creationDate DATETIME, locationIP STRING, browserUsed STRING, content STRING, length UINT) WITH primary_id_as_attribute="TRUE", STATS="outdegree_by_edgetype"
    CREATE GRAPH demo_graph (*)
'

gsql -g demo_graph '
    CREATE LOADING JOB load_Comment FOR GRAPH test_graph {
        DEFINE FILENAME file_Comment;
        LOAD file_Comment
            TO VERTEX Comment VALUES ($1, $0, $2, $3, $4, $5) USING header="true", separator="|";
 }
'

spark-shell --jars tigergraph-spark-connector-<version>-jar-with-dependencies.jar

logger.tg.name = com.tigergraph
logger.tg.level = info

val GRAPH = "demo_graph"
val URL = "http(s)://hostname:port[,http(s)://hostname:port]*"
val USERNAME = "tigergraph"
val PASSWORD = "tigergraph"
val LOADING_JOB = "load_Comment"
val FILENAME = "file_Comment"
val SEPARATOR = "|"

val df = spark.read.json("path/to/person.json")

df.write
    .format("tigergraph")
    .mode("append")
    .options(
        Map(
            "graph" -> GRAPH,
            "url" -> URL,
            "username" -> USERNAME,
            "password" -> PASSWORD,
            "loading.job" -> LOADING_JOB,
            "loading.filename" -> FILENAME,
            "loading.separator" -> SEPARATOR
        )
    )
    .save()

val df = spark.read.format("delta")
    .load("/path/to/delta/table")
    .select(
        "creationDate",
        "id",
        "locationIP",
        "browserUsed",
        "content",
        "length"
    )

val df = spark.table("catalog.db.table")
    .select(
        "creationDate",
        "id",
        "locationIP",
        "browserUsed",
        "content",
        "length"
    )

val df = spark.read
    .format("hudi")
    .load("/path/to/hudi/table")
    .select(
        "creationDate",
        "id",
        "locationIP",
        "browserUsed",
        "content",
        "length"
    )

24/01/22 16:15:45 INFO TigerGraphBatchWrite: Overall loading statistics: [ {
    "overall" : {
        "duration" : 15792,
        "size" : 48675207,
        "progress" : 0,
        "startTime" : 1706770863875,
        "averageSpeed" : 29546,
        "id" : "test_graph.load_Comment.spark.all.1706770859889",
        "endTime" : 1706770879667,
        "currentSpeed" : 29546,
        "statistics" : {
            "fileLevel" : {
                "validLine" : 466594,
                "notEnoughToken" : 0,
                "tokenExceedsBuffer" : 0,
                "rejectLine" : 0
            },
            "objectLevel" : {
                "vertex" : [ {
                "validObject" : 466593,
                "typeName" : "Comment",
                "invalidPrimaryId" : 1
                } ]
            }
        }
    },
    "workers" : [ {
        "tasks" : [ {
            "filename" : "file_Comment"
        } ]
    }, {
    "tasks" : [ {
        "filename" : "file_Comment"
        } ]
    } ]
} ]