Argthtjtr

I am just experimenting with Selenium Webdriver, and I am having a couple of issues happen that all seem related that I want to clear up before continuing. When I run the code pasted at the bottom, it takes forever for Selenium to do anything with the browser and it says "waiting for [insert site here]" for a while. Once those are done, then it finally finds the elements and interacts with the page. Sometimes I will get various errors and the errors aren't consistent (but the errors don't break the code it just takes longer until Selenium does what I asked it). And every now and then my antivirus pops up saying it blocked a website (global.ymtracking.com) because it was malware. I feel like this might be related to ads on the webpage and how Selenium loads the web page, but I don't really know since I am new to this. I've never had problems from just visiting this website.

using System;

using System.Collections.Generic;

using System.Drawing;

using System.IO;

using OpenQA.Selenium;

using OpenQA.Selenium.Chrome;



namespace WebScrappingTesting

{

    class Program

    {

        static IWebDriver driver;

        static Dictionary<string, Dictionary<string, float>> hockeyStats = new Dictionary<string, Dictionary<string, float>>();

        static string TeamName ="";

        static float TeamAge;

        static float Wins;

        static float Losses;



        static void Main(string args)

        {

            driver = new ChromeDriver();



            driver.Navigate().GoToUrl("https://www.hockey-reference.com/leagues/NHL_2019.html");



            using (StreamWriter file = new StreamWriter(@"C:UsershockeyData.csv", false))

            {

                file.WriteLine("Team Name,Avg Age,Wins,Losses");

            }



            for (var year = 2019; year > 2015; year--)

            {

                Console.WriteLine($"Getting data for {year}");



                var rowsOfData = driver.FindElements(By.CssSelector("#stats > tbody > tr"));



                for (var i = 1; i <= rowsOfData.Count; i++)

                {

                    TeamName = driver.FindElement(By.CssSelector($"#stats > tbody > tr:nth-child({i}) > td.left > a")).Text;

                    float.TryParse(driver.FindElement(By.CssSelector($"#stats > tbody > tr:nth-child({i}) > td:nth-child(3)")).Text, out TeamAge);

                    float.TryParse(driver.FindElement(By.CssSelector($"#stats > tbody > tr:nth-child({i}) > td:nth-child(5)")).Text, out Wins);

                    float.TryParse(driver.FindElement(By.CssSelector($"#stats > tbody > tr:nth-child({i}) > td:nth-child(6)")).Text, out Losses);

                    // TODO: need to change this to a 3 dim dictionary or an object to be able to also record the year of the data

                    hockeyStats[TeamName] = new Dictionary<string, float>() {

                        { "Age", TeamAge },

                        { "Wins", Wins },

                        { "Losses", Losses }

                    };

                }



                foreach (var item in hockeyStats)

                {

                    Console.WriteLine("=============================================================");

                    Console.WriteLine($"Team Name: {item.Key}, Avg Age: {item.Value["Age"]}");

                    Console.WriteLine($"Wins {item.Value["Wins"]}, Losses {item.Value["Losses"]}");

                }



                using (StreamWriter file = new StreamWriter(@"C:UsershockeyData.csv", true))

                {

                    foreach (var item in hockeyStats)

                    {

                        file.WriteLine($"{item.Key},{item.Value["Age"]},{item.Value["Wins"]},{item.Value["Losses"]}");

                    }

                }





                driver.FindElement(By.LinkText("Previous Season")).Click();

                //driver.FindElement(By.CssSelector($"[href*=/leagues/NHL_{year - 1}.html]"));





            }



            Console.WriteLine("=============================================================");

            Console.WriteLine("=============================================================");

            Console.WriteLine(@"You file has been saved to C:UsershockeyData.csv");

            Console.WriteLine("=============================================================");

            Console.WriteLine("=============================================================");



            driver.Close();

            driver.Dispose();

        }

    }

}

It looks like it was related to the website. I ran fiddler and there was a lot going on in the background of that site.