Why React js?

less than 1 minute read

Published:

Spark Find difference between two dates.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
from pyspark.sql.functions import *
from pyspark.sql.types import StructType, StructField, IntegerType, StringType,FloatType,TimestampType,DoubleType,DateType,ArrayType
values = [(1,"2019/07/01 23:00:01","2019/07/02 23:00:01"),
(2,"2019/07/01 12:40:32", "2019/07/08 23:00:01"),
(3,"2019/07/01 09:54:00", "2019/07/08 23:00:01"),
(4,"2019/07/01 10:12:43", "2019/07/08 23:00:01")]
columns = ['Key', 'LATEST_POLL_TIME', 'F_START_TIME']		  
df = spark.createDataFrame(values, columns)
format = "yyyy/MM/dd HH:mm:ss"
df = df.withColumn('LATEST_POLL_TIME',unix_timestamp(df.LATEST_POLL_TIME, format).cast('timestamp'))
df = df.withColumn('F_START_TIME',unix_timestamp(df.F_START_TIME, format).cast('timestamp'))
df = df.withColumn("seconds",  unix_timestamp(df.F_START_TIME, format) - unix_timestamp(df.LATEST_POLL_TIME, format))

df = df.withColumn("minutes",  int(round(df.seconds/60)))
df = df.withColumn("hours",  int(round(df.minutes/60)))
df.show(10)

list1 = ['a','b','c','d','e','f','g','h', 'i', 'j', 'k', 'l']
print(list1[:10])

Reference Stackoverflow forum link