Python inserts newline by writing to csv

Question

I am trying to scrape http://www.the-numbers.com/movie/budgets/all

but when I write the table into a csv file, there is an additional line with the counter index written in between each movie row... how can I get rid of this? I dont understand how that counter line is being written to the csv...

import csv,os
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen, URLError
from selenium import webdriver

counter = 0
currentDir=os.getcwd()
filename = currentDir + "\\theNumbersScraper.csv"

pagecount = 1
headers=['ID',
 'Release Date',
 'Movie',
 'Production Budget',
 'Domestic Gross',
 'Worldwide Gross']

with open(filename, 'w' ,newline='\n',encoding='utf-8') as csvfile:
        #writer = csv.DictWriter(csvfile, fieldnames=dictionary)#write headers
        #writer.writeheader()    
        #csvfile = open(filename, 'w', newline='',encoding='utf-8')
    writer = csv.writer(csvfile,delimiter='|')
    writer.writerow(headers)

#with open(filename, 'a', newline='',encoding='utf-8') as csvfile:
    #writer = csv.DictWriter(csvfile, fieldnames=dictionary)#write headers
    #writer.writeheader()    
    #csvfile = open(filename, 'w', newline='',encoding='utf-8')
#    writer = csv.writer(csvfile,delimiter='|')

    while pagecount <5401:

        """
        #movie-entries go from http://www.the-numbers.com/movie/budgets/all/1
        #to http://www.the-numbers.com/movie/budgets/all/5401
        #so there are 5400 entries
        """
        request = Request("http://www.the-numbers.com/movie/budgets/all/"+str(pagecount))
        request.add_header('User-agent', 'wswp')
        website = urlopen(request).read().strip()

        soup = BeautifulSoup(website,'lxml')

        """#obsolete
        headertags = soup.find("table").find_next("tr").find_all("th")

        headers= []
        for line in headertags:
            headers.append(line.string)

        headers[0] = 'ID'

        """

        #movie-entries go from http://www.the-numbers.com/movie/budgets/all/1
        #to http://www.the-numbers.com/movie/budgets/all/5401
        #so there are 5400 entries

        all_tr = soup.find_all("tr")
        for movie in range(1, len(all_tr)):
            row=[]
            counter+=1
            row.append(counter)
            td = all_tr[movie].find_all("td")            
            for colIndex in range(1, len(td)):                
                row.append(td[colIndex].string)
            writer.writerow(row)


        """
        for tr in all_tr:
            row = []
            td = tr.find_all("td")
            for i in range(1, 4):
                row.append(str(counter))
                row.append(td[i].string)
            writer.writerow(row)
            counter+=1
        """
        pagecount +=100


csvfile.close()

The part of interest is actually this block only:

all_tr = soup.find_all("tr")
for movie in range(1, len(all_tr)):
    row=[]
    counter+=1
    row.append(counter)
    td = all_tr[movie].find_all("td")            
    for colIndex in range(1, len(td)):                
        row.append(td[colIndex].string)
    writer.writerow(row)

The output is:

ID|Release Date|Movie|Production Budget|Domestic Gross|Worldwide Gross
1|12/18/2009|Avatar|$425,000,000|$760,507,625|$2,783,918,982
2
3|5/20/2011|Pirates of the Caribbean: On Stranger      Tides|$410,600,000|$241,063,875|$1,045,663,875
4
5|5/1/2015|Avengers: Age of Ultron|$330,600,000|$459,005,868|$1,408,218,722
6

and so on, with additional counter lines between the output that I don't want, how can I get rid of that? (and where does it come from?)

Answer 1

Check the source code of http://www.the-numbers.com/movie/budgets/all/5401

In that table, every second row is an empty <tr> tag.

Answer 2

ok so every second line was an empty tag, I adapted my code accordingly:

    all_tr = soup.find_all("tr")
    for movie in range(1, len(all_tr)):
        if movie % 2 == 0:
            continue
        else:
            row=[]
            counter+=1
            row.append(counter)
            td = all_tr[movie].find_all("td")            
            for colIndex in range(1, len(td)):                
                row.append(td[colIndex].string)
            writer.writerow(row)