df.sort_values("col")

df.sort(desc(col))

df.to_csv(file_name)

df.repartition(1).write.mode('overwrite').option("header","true").csv(file_name)

df.to_json(file_name,orient='records',lines=True)

df.repartition(1).write.mode('overwrite').option("header","true").json(file_name)

df.rename('old','new')

df.withColumnRenamed('old','new')

pd.read_csv('file',sep=',')

spark.read.csv('file',sep=',')

pd.read_json('file',lines=True)

spark.read.json('file',lines=True)

df.shape[0]

df.count()

df.shape[1]

len(df.columns)

df[~df['col'].contains('pattern')]

df.filter("col not like '%pattern%'")

df[df['col'].isin([a,b,c])]

from pyspark.sql.function import col; df.filter(~col('bar').isin(['a','b']))

df['new_col']= 'abc'

from pyspark.sql.function import lit; df.withColumn('new_col',lit('abc'))

df[['a','b']] 

df[['a','b']]

df['col'].nunique()

df.select('col').distinct()

df1.select("sentence").subtract(df2.select("sentence")).distinct()

df1.select("sentence").intersect(df2.select("sentence")).distinct()

df1.select("sentence").union(df2.select("sentence")).distinct()

(df['col'] == 'TRUE').astype('i2')

(df['col'] == 'TRUE').astype(int)

spark.sql('SELECT datediff(all.timestamp2,all.timestamp1)*24*60*60 + (hour(all.timestamp2)- hour(all.timestamp1))*60*60 + (minute(all.timestamp2) - minute(all.timestamp1))*60 + (second(all.timestamp2) - second(all.timestamp1)) as tol_time_diff from df')

def test_sum(series):
	l = series.values.tolist()
	new_l = [en for en in l if en!='']
	return ';'.join(new_l)
target = x.groupby('uid').agg({'str':test_sum})

df['col'].tolist()