update to download first 251 pages. Filter the ranked games from there. Sort the results with respect to rank. Also include a new json for the object ids of the ranked games.

Author: mertaybat

.gitignore

@@ -7,4 +7,4 @@ html-pages
 xmls
 credentials.properties
 node_modules
-
+.angular

README.md

@@ -47,8 +47,8 @@ In order not to generate too much load on boardgamegeek, every XML API call is p
 recommended and required by boardgamegeek.
 
 The script will first create some necessary directories and create archives of your old runs.
-Then it will first download the first 249 html pages of ranked games from boardgamegeek.com. It will 
-then generate a list of game ids that correspond to those 24900 games (every page contains 100 games).
+Then it will first download the first 251 html pages of ranked games from boardgamegeek.com. It will 
+then generate a list of game ids that correspond to those games that have a rank.
 It will then download using the boardgamegeek XML api v.1 to download the xml data of these games. Finally it will 
 run a small software program contained in the converter directory to convert all the xml data to json. It will generate
 2 json files; one maps 1-1 to the xml data and another which is custom format used by the frontend application.

converter/app/src/main/kotlin/converter/App.kt

@@ -15,44 +15,54 @@ fun main() {
     objectMapper.setSerializationInclusion(NON_EMPTY)
     val listOfBoardgames: MutableList<Boardgame> = mutableListOf()
     val listOfCustomResults: MutableList<CustomBoardgame> = mutableListOf()
+    val listOfIds: MutableList<Identifier> = mutableListOf()
     File("./").listFiles()?.filter { it.isFile }?.filter { it.extension == "xml" }?.forEach {
         val boardgames = unmarshaller.unmarshal(it.inputStream()) as Boardgames
-        boardgames.boardgame.forEach {
-            listOfBoardgames.add(it)
-            listOfCustomResults.add(it.toInternal())
-        }
+        boardgames.boardgame
+                .filter { it.statistics.ratings.ranks.rank.none { it.valueAttribute == "Not Ranked" } }
+                .forEach {
+                    listOfBoardgames.add(it)
+                    val customBoardgame = it.toInternal()
+                    listOfCustomResults.add(customBoardgame)
+                    listOfIds.add(Identifier(customBoardgame.objectId, customBoardgame.name, customBoardgame.boardGameRank))
+                }
     }
     val writer = FileWriter("./boardgames.json")
-    writer.write(objectMapper.writeValueAsString(listOfBoardgames))
+    writer.write(objectMapper.writeValueAsString(listOfBoardgames.sortedBy { it.toRank() }))
     writer.flush()
     writer.close()
     val writer2 = FileWriter("./boardgames-custom.json")
-    writer2.write(objectMapper.writeValueAsString(listOfCustomResults))
+    writer2.write(objectMapper.writeValueAsString(listOfCustomResults.sortedBy{ it.boardGameRank }))
     writer2.flush()
     writer2.close()
+    val writer3 = FileWriter("./object-ids.json")
+    writer3.write(objectMapper.writeValueAsString(listOfIds.sortedBy {it.boardGameRank }))
+    writer3.flush()
+    writer3.close()
 }
 
 private fun Boardgame.toInternal(): CustomBoardgame {
     return CustomBoardgame(
-        objectId = this.objectid,
-        yearPublished = this.yearpublished.toInt(),
-        minPlayers = this.minplayers.toInt(),
-        maxPlayers = this.maxplayers.toInt(),
-        playingTime = this.playingtime.toInt(),
-        minPlayTime = this.minplaytime.toInt(),
-        maxPlayTime = this.maxplaytime.toInt(),
-        age = this.age.toInt(),
-        name = this.name.singleOrNull { it.primary == "true" }?.value ?: this.name.first().value,
-        description = this.description,
-        thumbnailImageLink = this.thumbnail,
-        publisher = this.boardgamepublisher?.value,
-        version = this.boardgameversion?.value,
-        category = this.boardgamecategory?.value,
-        families = this.boardgamefamily?.mapNotNull { it.value } ?: emptyList(),
-        gameMechanics = this.boardgamemechanic?.value,
-        bestPlayedWith = this.poll.single { it.name == "suggested_numplayers"}.results.sortedBy {
-            it.result.single { it.valueAttribute == "Best" }.numvotes.toInt()
-        }.last().numplayers,
-        boardGameRank = this.statistics.ratings.ranks.rank.single { it.name == "boardgame" }.valueAttribute.toInt()
+            objectId = this.objectid,
+            yearPublished = this.yearpublished.toInt(),
+            minPlayers = this.minplayers,
+            maxPlayers = this.maxplayers,
+            playingTime = this.playingtime,
+            minPlayTime = this.minplaytime,
+            maxPlayTime = this.maxplaytime,
+            age = this.age,
+            name = this.name.singleOrNull { it.primary == "true" }?.value ?: this.name.first().value,
+            description = this.description,
+            thumbnailImageLink = this.thumbnail,
+            publisher = this.boardgamepublisher?.value,
+            version = this.boardgameversion?.value,
+            category = this.boardgamecategory?.value,
+            families = this.boardgamefamily?.mapNotNull { it.value } ?: emptyList(),
+            gameMechanics = this.boardgamemechanic?.value,
+            bestPlayedWith = this.poll.single { it.name == "suggested_numplayers" }.results.sortedBy {
+                it.result.single { it.valueAttribute == "Best" }.numvotes.toInt()
+            }.last().numplayers,
+            boardGameRank = this.toRank()
     )
 }
+private fun Boardgame.toRank(): Int = this.statistics.ratings.ranks.rank.single { it.name == "boardgame" }.valueAttribute.toInt()

converter/app/src/main/kotlin/converter/Identifier.kt

@@ -0,0 +1,7 @@
+package converter
+
+data class Identifier(
+        val objectId: Int,
+        val name: String,
+        val boardGameRank: Int
+)

converter/app/src/main/schema/schema.xsd

@@ -27,7 +27,7 @@
           <xs:attribute type="xs:string" name="name" use="optional"/>
           <xs:attribute type="xs:string" name="friendlyname" use="optional"/>
           <xs:attribute type="xs:string" name="valueAttribute" use="optional"/>
-          <xs:attribute type="xs:float" name="bayesaverage" use="optional"/>
+          <xs:attribute type="xs:string" name="bayesaverage" use="optional"/>
         </xs:extension>
       </xs:simpleContent>
     </xs:complexType>

create-data.sh

@@ -50,14 +50,14 @@ cd ../
 
 # download all html data create ids
 cd html-pages
-for i in {1..249}; do curl https://boardgamegeek.com/browse/boardgame/page/$i -o $i.html --compressed -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" -H "Accept-Language: en-US,en;q=0.5" -H "Accept-Encoding: gzip, deflate, br" -H "DNT: 1" -H "Alt-Used: boardgamegeek.com" -H "Connection: keep-alive" -H "Cookie: bggusername=$username; bggpassword=$password" -H "Upgrade-Insecure-Requests: 1" -H "Sec-Fetch-Dest: document" -H "Sec-Fetch-Mode: navigate" -H "Sec-Fetch-Site: none" -H "Sec-Fetch-User: ?1" -H "TE: trailers"; done
-ag -o "href=\"/boardgame/\d{1,10}/.*?\"\s>" | awk -F "/" '{print $3}' > ids.txt
+for i in {1..251}; do curl https://boardgamegeek.com/browse/boardgame/page/$i -o $i.html --compressed -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" -H "Accept-Language: en-US,en;q=0.5" -H "Accept-Encoding: gzip, deflate, br" -H "DNT: 1" -H "Alt-Used: boardgamegeek.com" -H "Connection: keep-alive" -H "Cookie: bggusername=$username; bggpassword=$password" -H "Upgrade-Insecure-Requests: 1" -H "Sec-Fetch-Dest: document" -H "Sec-Fetch-Mode: navigate" -H "Sec-Fetch-Site: none" -H "Sec-Fetch-User: ?1" -H "TE: trailers"; done
+ag -o "href=\"/(boardgame|boardgameexpansion)/\d{1,10}/.*?\"\s>" | awk -F "/" '{print $3}' > ids.txt
 cd ../
 
 
 # download all xmls
 cd xmls
-for i in {1..249}
+for i in {1..251}
 do
         let start="1+($i-1)*100"
         let end="$i*100"