I transformed the existing code which was in python pasted below was in pyspark. 

Python code:

    import json
    import csv
    
    
    def main():
    	# create a simple JSON array
        with open('paytm_tweets_data_1495614657.json') as str:
    
            tweetsList = []
            # change the JSON string into a JSON object
            jsonObject = json.load(str)
    
            #print(jsonObject)
    
            # # print the keys and values
            for i in range(len(jsonObject)):
            	tweetsList.insert(i,jsonObject[i]["text"])
    
            #print(tweetsList)
        displaySentiment(tweetsList)
    
    
    
    def displaySentiment(tweetsList):
        aDict = {}
    
        from sentiment import sentiment_score
    
        for i in range(len(tweetsList)):
            aDict[tweetsList[i]] = sentiment_score(tweetsList[i])
        print (aDict)
    
    
        with open('PaytmtweetSentiment.csv', 'w') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames = ["Tweets", "Sentiment Value"])
            writer.writeheader()
            writer = csv.writer(csv_file)
            for key, value in aDict.items():
                writer.writerow([key, value])
    
    
    if __name__ == '__main__':
        main()

Converted Pyspark Code:

    import json
    import csv
    import os
    from pyspark import SparkContext, SparkConf
    from pyspark.python.pyspark.shell import spark
    
    os.environ['PYSPARK_PYTHON'] = "/usr/local/bin/python3"
    
    
    def main():
        path = "/Users/i322865/DeepInsights/bitbucket-code/ai-engine/twitter-sentiment-analysis/flipkart_tweets_data_1495601666.json"
        peopleDF = spark.read.json(path).rdd
        df = peopleDF.map(lambda row: row['text'])
        print(df.collect())
        displaySentiment(df.collect())
    
    
    
    def displaySentiment(tweetsList):
        from sentiment import sentiment_score
    
        aDict = sentiment_score(tweetsList)
    
        #
        with open('paytmtweetSentiment.csv', 'w') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames = ["Tweets", "Sentiment Value"])
            writer.writeheader()
            writer = csv.writer(csv_file)
            for i in range(len(tweetsList)):
                writer.writerow([tweetsList[i], aDict[i]])
                print([tweetsList[i], aDict[i]])
    
    
    if __name__ == '__main__':
        conf = SparkConf().setAppName("Test").setMaster("local")
        sc = SparkContext.getOrCreate(conf=conf)
        main()


I ran both programs but didn't see any significant performance improvement. What am I missing? Please could you shed some thoughts?