Update the advanced player box score parsing (#291)

Author: jaebradley

basketball_reference_web_scraper/html.py

@@ -249,12 +249,13 @@ def __init__(self, html):
     @property
     def rows_query(self):
         return """
-            //table[@id="advanced_stats"]
+            //table[@id="advanced"]
             /tbody
             /tr[
-                contains(@class, "full_table") or 
-                contains(@class, "italic_text partial_table") 
-                and not(contains(@class, "rowSum"))
+                (
+                    not(contains(@class, 'thead'))
+                    or not(contains(@class, "rowSum"))
+                )
             ]
         """
 
@@ -306,6 +307,31 @@ class PlayerAdvancedSeasonTotalsRow(PlayerIdentificationRow):
     def __init__(self, html):
         super().__init__(html=html)
 
+    @property
+    def player_cell(self):
+        cells = self.html.xpath('td[@data-stat="name_display"]')
+
+        if len(cells) > 0:
+            return cells[0]
+
+        return None
+
+    @property
+    def slug(self):
+        cell = self.player_cell
+        if cell is None:
+            return ''
+
+        return cell.get('data-append-csv')
+
+    @property
+    def name(self):
+        cell = self.player_cell
+        if cell is None:
+            return ''
+
+        return cell.text_content()
+
     @property
     def position_abbreviations(self):
         cells = self.html.xpath('td[@data-stat="pos"]')
@@ -326,7 +352,7 @@ def age(self):
 
     @property
     def team_abbreviation(self):
-        cells = self.html.xpath('td[@data-stat="team_id"]')
+        cells = self.html.xpath('td[@data-stat="team_name_abbr"]')
 
         if len(cells) > 0:
             return cells[0].text_content()
@@ -533,7 +559,10 @@ def value_over_replacement_player(self):
 
     @property
     def is_combined_totals(self):
-        return self.team_abbreviation == "TOT"
+        #  No longer says 'TOT' - now says 2TM, 3TM, etc.
+        # Can safely use the 'TM' suffix as an identifier as no team abbreviations
+        # end in 'TM'
+        return self.team_abbreviation.endswith("TM")
 
 
 class PlayerSeasonTotalsRow:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "basketball_reference_web_scraper"
-version = "4.15.0"
+version = "4.15.1"
 description = "A Basketball Reference client that generates data by scraping the website"
 authors = ["Jae Bradley <jae.b.bradley@gmail.com>"]
 maintainers = ["Jae Bradley <jae.b.bradley@gmail.com>"]

scripts/build.sh

@@ -41,14 +41,12 @@ function main() {
   # TODO: @jaebradley re-enable integration pytests
   "${poetry_program_path}" run coverage run --source=basketball_reference_web_scraper --module pytest \
     --ignore "./tests/integration/client/test_player_season_totals.py" \
-    --ignore "./tests/integration/client/test_players_advanced_season_totals.py" \
     --ignore "./tests/integration/client/test_players_season_totals.py" \
     --ignore "./tests/integration/client/test_search.py" \
     --ignore "./tests/integration/client/test_season_schedule.py" \
     --ignore "./tests/integration/client/test_standings.py" \
     --ignore "./tests/integration/client/test_team_box_scores.py" \
     --ignore "./tests/integration/client/test_client.py" \
-    --ignore "./tests/integration/client/test_player_regular_season_box_scores.py" \
     --ignore "./tests/integration/parsers" \
     --ignore "./tests/integration/html/test_daily_box_scores_page.py"
 

tests/integration/client/output/expected/player_box_scores/2019/bradlav01/exclude_inactive.csv

@@ -0,0 +1,64 @@
+active,date,points_scored,plus_minus,team,location,opponent,outcome,seconds_played,made_field_goals,attempted_field_goals,made_three_point_field_goals,attempted_three_point_field_goals,made_free_throws,attempted_free_throws,offensive_rebounds,defensive_rebounds,assists,steals,blocks,turnovers,personal_fouls,game_score
+True,2018-10-17,8,-9,LOS ANGELES CLIPPERS,HOME,DENVER NUGGETS,LOSS,1676,4,10,0,2,0,0,2,0,0,1,2,2,2,3.6
+True,2018-10-19,3,0,LOS ANGELES CLIPPERS,HOME,OKLAHOMA CITY THUNDER,WIN,1732,1,8,1,5,0,0,1,1,2,1,0,1,3,-1.0
+True,2018-10-21,4,-21,LOS ANGELES CLIPPERS,HOME,HOUSTON ROCKETS,WIN,1447,2,5,0,1,0,0,0,1,3,1,0,0,3,3.5
+True,2018-10-23,2,-6,LOS ANGELES CLIPPERS,AWAY,NEW ORLEANS PELICANS,LOSS,1710,1,8,0,2,0,0,1,1,0,0,0,1,3,-4.4
+True,2018-10-26,14,10,LOS ANGELES CLIPPERS,AWAY,HOUSTON ROCKETS,WIN,1497,6,10,1,3,1,1,0,3,0,1,0,2,3,8.1
+True,2018-10-28,15,13,LOS ANGELES CLIPPERS,HOME,WASHINGTON WIZARDS,WIN,1390,6,12,3,6,0,0,1,0,5,0,1,2,4,10.3
+True,2018-10-30,9,-11,LOS ANGELES CLIPPERS,AWAY,OKLAHOMA CITY THUNDER,LOSS,1809,4,10,1,4,0,0,0,3,2,1,0,2,5,2.9
+True,2018-11-01,2,4,LOS ANGELES CLIPPERS,AWAY,PHILADELPHIA 76ERS,LOSS,2180,1,6,0,1,0,0,1,1,1,1,0,2,1,-1.5
+True,2018-11-02,7,8,LOS ANGELES CLIPPERS,AWAY,ORLANDO MAGIC,WIN,1388,3,11,1,3,0,0,1,0,1,0,0,2,1,-0.5
+True,2018-11-19,5,-2,LOS ANGELES CLIPPERS,AWAY,ATLANTA HAWKS,WIN,1170,2,7,0,2,1,1,0,0,0,1,0,0,4,0.3
+True,2018-11-20,4,-8,LOS ANGELES CLIPPERS,AWAY,WASHINGTON WIZARDS,LOSS,1914,2,3,0,1,0,0,0,1,0,0,0,3,3,-1.2
+True,2018-11-23,13,3,LOS ANGELES CLIPPERS,HOME,MEMPHIS GRIZZLIES,WIN,1977,5,8,0,1,3,4,0,2,3,1,1,0,3,12.2
+True,2018-11-25,4,21,LOS ANGELES CLIPPERS,AWAY,PORTLAND TRAIL BLAZERS,WIN,2013,2,11,0,4,0,1,0,4,2,3,1,0,2,2.2
+True,2018-11-28,0,26,LOS ANGELES CLIPPERS,HOME,PHOENIX SUNS,WIN,1726,0,7,0,2,0,0,0,2,0,0,2,1,3,-5.1
+True,2018-11-29,16,26,LOS ANGELES CLIPPERS,AWAY,SACRAMENTO KINGS,WIN,1900,6,10,4,7,0,0,1,2,6,0,1,3,4,13.0
+True,2018-12-02,12,-7,LOS ANGELES CLIPPERS,AWAY,DALLAS MAVERICKS,LOSS,2175,5,11,2,3,0,0,0,3,3,2,0,1,3,9.1
+True,2018-12-03,15,-9,LOS ANGELES CLIPPERS,AWAY,NEW ORLEANS PELICANS,WIN,1757,7,12,1,4,0,0,1,2,2,1,1,3,5,8.8
+True,2018-12-05,5,-15,LOS ANGELES CLIPPERS,AWAY,MEMPHIS GRIZZLIES,LOSS,1960,2,8,1,3,0,0,0,5,1,1,0,1,2,1.6
+True,2018-12-08,7,-21,LOS ANGELES CLIPPERS,HOME,MIAMI HEAT,LOSS,2204,3,6,1,3,0,0,0,4,1,0,1,1,3,4.4
+True,2018-12-10,8,5,LOS ANGELES CLIPPERS,AWAY,PHOENIX SUNS,WIN,2246,3,8,0,3,2,2,1,0,2,0,0,2,1,3.3
+True,2018-12-11,0,-28,LOS ANGELES CLIPPERS,HOME,TORONTO RAPTORS,LOSS,1313,0,6,0,3,0,0,0,0,1,0,0,0,1,-3.9
+True,2018-12-13,15,-16,LOS ANGELES CLIPPERS,AWAY,SAN ANTONIO SPURS,LOSS,1704,5,9,3,4,2,2,0,3,3,0,0,1,1,12.3
+True,2018-12-15,10,4,LOS ANGELES CLIPPERS,AWAY,OKLAHOMA CITY THUNDER,LOSS,2062,4,9,2,6,0,0,2,5,4,1,1,6,3,5.5
+True,2018-12-17,0,-22,LOS ANGELES CLIPPERS,HOME,PORTLAND TRAIL BLAZERS,LOSS,1802,0,4,0,3,0,0,1,0,0,0,0,1,5,-5.1
+True,2018-12-20,2,-4,LOS ANGELES CLIPPERS,HOME,DALLAS MAVERICKS,WIN,1662,1,4,0,2,0,0,0,1,2,0,0,0,5,-0.7
+True,2018-12-22,9,17,LOS ANGELES CLIPPERS,HOME,DENVER NUGGETS,WIN,1323,4,6,1,1,0,0,0,1,3,1,1,1,3,8.3
+True,2018-12-23,12,-4,LOS ANGELES CLIPPERS,AWAY,GOLDEN STATE WARRIORS,LOSS,1863,4,8,2,3,2,2,1,2,2,0,0,1,4,8.1
+True,2018-12-26,15,14,LOS ANGELES CLIPPERS,HOME,SACRAMENTO KINGS,WIN,1820,6,11,2,5,1,2,2,3,4,0,0,1,2,12.6
+True,2018-12-28,8,-2,LOS ANGELES CLIPPERS,AWAY,LOS ANGELES LAKERS,WIN,1746,4,12,0,5,0,0,1,4,3,1,0,1,4,3.6
+True,2018-12-29,10,-18,LOS ANGELES CLIPPERS,HOME,SAN ANTONIO SPURS,LOSS,1951,4,9,2,2,0,0,3,4,4,0,0,1,3,9.2
+True,2019-01-01,6,-14,LOS ANGELES CLIPPERS,HOME,PHILADELPHIA 76ERS,LOSS,1749,3,7,0,1,0,0,0,0,1,0,0,0,2,2.2
+True,2019-01-04,2,4,LOS ANGELES CLIPPERS,AWAY,PHOENIX SUNS,WIN,1219,1,4,0,2,0,0,0,1,2,0,1,1,5,-1.0
+True,2019-01-06,7,2,LOS ANGELES CLIPPERS,HOME,ORLANDO MAGIC,WIN,1513,3,10,1,2,0,0,1,1,0,1,0,0,1,2.8
+True,2019-01-08,10,4,LOS ANGELES CLIPPERS,HOME,CHARLOTTE HORNETS,WIN,1755,3,9,2,3,2,2,0,1,1,0,1,1,2,4.8
+True,2019-01-10,2,-11,LOS ANGELES CLIPPERS,AWAY,DENVER NUGGETS,LOSS,1549,1,6,0,4,0,0,0,2,2,2,0,0,1,1.8
+True,2019-01-12,3,-9,LOS ANGELES CLIPPERS,HOME,DETROIT PISTONS,LOSS,1323,1,3,1,1,0,0,0,0,2,0,0,0,1,2.3
+True,2019-01-14,9,-13,LOS ANGELES CLIPPERS,HOME,NEW ORLEANS PELICANS,LOSS,1796,3,9,3,8,0,0,0,2,5,0,0,2,1,5.6
+True,2019-01-16,15,-21,LOS ANGELES CLIPPERS,HOME,UTAH JAZZ,LOSS,1496,6,10,3,4,0,0,1,2,1,0,0,2,3,9.2
+True,2019-01-18,6,-11,LOS ANGELES CLIPPERS,HOME,GOLDEN STATE WARRIORS,LOSS,1981,3,13,0,4,0,0,3,1,0,0,0,2,5,-3.5
+True,2019-01-20,15,-2,LOS ANGELES CLIPPERS,AWAY,SAN ANTONIO SPURS,WIN,2479,6,15,3,6,0,0,0,5,4,1,0,2,2,9.4
+True,2019-01-22,15,5,LOS ANGELES CLIPPERS,AWAY,DALLAS MAVERICKS,LOSS,2310,5,11,3,6,2,2,2,3,3,0,0,1,2,11.9
+True,2019-01-23,12,10,LOS ANGELES CLIPPERS,AWAY,MIAMI HEAT,WIN,2084,5,11,2,6,0,0,2,0,3,1,0,1,2,9.0
+True,2019-01-25,11,8,LOS ANGELES CLIPPERS,AWAY,CHICAGO BULLS,WIN,2118,5,12,1,2,0,0,1,4,2,1,0,2,2,6.1
+True,2019-01-27,12,14,LOS ANGELES CLIPPERS,HOME,SACRAMENTO KINGS,WIN,2192,4,8,2,3,2,2,1,5,6,1,0,2,4,11.8
+True,2019-01-28,12,11,LOS ANGELES CLIPPERS,HOME,ATLANTA HAWKS,LOSS,1768,4,10,4,8,0,0,1,1,0,0,0,1,5,4.6
+True,2019-01-31,13,3,LOS ANGELES CLIPPERS,HOME,LOS ANGELES LAKERS,LOSS,2124,5,15,2,9,1,2,2,6,1,2,1,1,1,9.3
+True,2019-02-02,6,10,LOS ANGELES CLIPPERS,AWAY,DETROIT PISTONS,WIN,1526,2,7,2,4,0,0,1,0,2,0,0,0,0,4.0
+True,2019-02-03,4,-7,LOS ANGELES CLIPPERS,AWAY,TORONTO RAPTORS,LOSS,1295,2,6,0,3,0,0,0,2,0,0,0,0,2,0.4
+True,2019-02-05,6,5,LOS ANGELES CLIPPERS,AWAY,CHARLOTTE HORNETS,WIN,2396,2,5,1,2,1,2,0,2,1,0,1,1,3,2.7
+True,2019-02-09,8,-6,MEMPHIS GRIZZLIES,HOME,NEW ORLEANS PELICANS,WIN,1486,3,10,2,4,0,0,0,1,2,1,0,3,3,0.7
+True,2019-02-12,33,-1,MEMPHIS GRIZZLIES,HOME,SAN ANTONIO SPURS,LOSS,2274,15,21,3,5,0,0,0,6,6,1,0,2,1,28.9
+True,2019-02-13,15,-24,MEMPHIS GRIZZLIES,AWAY,CHICAGO BULLS,LOSS,1719,7,12,1,2,0,0,0,1,5,1,0,2,1,11.8
+True,2019-02-22,17,-1,MEMPHIS GRIZZLIES,HOME,LOS ANGELES CLIPPERS,LOSS,1964,7,14,1,3,2,2,1,2,1,2,0,2,5,10.0
+True,2019-02-23,4,-19,MEMPHIS GRIZZLIES,AWAY,CLEVELAND CAVALIERS,LOSS,1134,1,5,0,2,2,3,0,1,2,0,0,2,4,-1.4
+True,2019-02-25,15,2,MEMPHIS GRIZZLIES,HOME,LOS ANGELES LAKERS,WIN,2020,5,14,3,8,2,2,1,5,3,1,0,2,3,9.3
+True,2019-02-27,23,-1,MEMPHIS GRIZZLIES,HOME,CHICAGO BULLS,LOSS,2258,7,18,3,7,6,6,3,4,7,1,0,1,1,21.0
+True,2019-03-02,15,25,MEMPHIS GRIZZLIES,AWAY,DALLAS MAVERICKS,WIN,1923,5,12,3,6,2,2,1,4,5,0,0,0,3,12.8
+True,2019-03-03,27,3,MEMPHIS GRIZZLIES,AWAY,OKLAHOMA CITY THUNDER,LOSS,2128,11,19,2,6,3,3,0,5,7,2,0,3,3,22.3
+True,2019-03-05,14,5,MEMPHIS GRIZZLIES,HOME,PORTLAND TRAIL BLAZERS,WIN,2058,5,14,3,8,1,2,0,3,3,2,0,2,3,7.6
+True,2019-03-08,7,10,MEMPHIS GRIZZLIES,HOME,UTAH JAZZ,WIN,2148,2,7,1,1,2,2,0,1,6,1,0,3,1,5.0
+True,2019-03-10,21,26,MEMPHIS GRIZZLIES,HOME,ORLANDO MAGIC,WIN,2244,8,17,3,8,2,2,0,2,3,2,0,2,1,14.6
+True,2019-03-13,5,-27,MEMPHIS GRIZZLIES,AWAY,ATLANTA HAWKS,LOSS,1155,2,8,0,4,1,1,1,0,1,0,0,2,2,-1.2
+True,2019-03-16,21,-9,MEMPHIS GRIZZLIES,AWAY,WASHINGTON WIZARDS,LOSS,2022,9,17,3,9,0,0,1,1,5,0,0,2,3,14.0