Add new data

warioddly · May 14, 2024 · 10e5bf3 · 10e5bf3
1 parent 5881298
commit 10e5bf3
Show file tree

Hide file tree

Showing 12 changed files with 23 additions and 15 deletions.
diff --git a/parser/bin/parser.dart b/parser/bin/parser.dart
@@ -69,11 +69,11 @@ Future<void> _scrapy() async {
 
   scrapy.visited.addAll({
   'https://www.sanjyra.net/man/1',
-  'https://www.sanjyra.net/man/2',
-  'https://www.sanjyra.net/man/32252'
+  'https://www.sanjyra.net/man/32252',
+  'https://www.sanjyra.net/man/44275'
   });
 
-  await scrapy.parse(['', 'https://www.sanjyra.net/man/44275']);
+  await scrapy.parse(['1', 'https://www.sanjyra.net/man/2']);
 
   print('\n[+] Done! Parsed ${scrapy.people.length} people.');
 

diff --git a/parser/lib/scrapy.dart b/parser/lib/scrapy.dart
@@ -9,9 +9,11 @@ import 'package:parser/people_model.dart';
 class Scrapy {
 
   final visited = <String>{};
+  final queue = <String>{};
   final people = <People>[];
   final bundler = Bundler();
 
+
   Future<void> parse(List<String> args) async {
 
     final parent = args[0];
@@ -28,30 +30,33 @@ class Scrapy {
     }
 
     final document = parser.parse(response.body);
-    List<String> children = _getChildren(document);
+    final children = _getChildren(document);
 
     final person = _getPeople(parent, url, document);
 
     people.add(person);
-
     bundler.generate('${jsonEncode(person.toJson())},', 'dynamic_all_data.json', FileMode.append);
 
     if (children.isNotEmpty) {
       final id = _getId(url);
 
-      for (int i = 0; i < children.length; i++) {
-        await parse([id, children[i]]);
+      queue.addAll(children);
+
+      for (final child in children) {
+        await parse([id, child]);
       }
 
+      queue.removeWhere((element) => children.contains(element));
+
     }
 
 
   }
 
 
-  List<String> _getChildren(Document document) {
+  Set<String> _getChildren(Document document) {
 
-    final urls = <String>[];
+    final urls = <String>{};
 
     final elements = document.querySelector('#app > main > div > div.container.mx-auto.w-full > div.p-3.flex.flex-nowrap.items-start.text-center.mx-auto.w-max.max-w-full.overflow-x-auto > table > tbody');
 
@@ -69,7 +74,7 @@ class Scrapy {
 
           var url = child.children.first.attributes['href'];
 
-          if (url != null && !visited.contains(url)) {
+          if (url != null && !visited.contains(url) && !queue.contains(url)) {
             urls.add(url);
           }
 

diff --git a/parser/output/Куу уул/all_data.json b/parser/output/Куу уул/all_data.json
diff --git a/parser/output/Куу уул/all_data_site.json b/parser/output/Куу уул/all_data_site.json
diff --git a/parser/output/куу уул/all_data.json → parser/output/Куу уул/dynamic_all_data.json b/parser/output/куу уул/all_data.json → parser/output/Куу уул/dynamic_all_data.json
diff --git a/parser/output/Кызыл уул/all_data.json b/parser/output/Кызыл уул/all_data.json
@@ -0,0 +1 @@
+[{"id":"44275","source":"1","name":"Кызыл уул","fullName":"Долон бий уулу Кызыл уул","bio":"-","url":"https://www.sanjyra.net/man/44275"},{"id":"44276","source":"44275","name":"Каңды","fullName":"Кызыл уул уулу Каңды","bio":"-","url":"https://www.sanjyra.net/man/44276"},{"id":"44277","source":"44275","name":"Дөөлөс","fullName":"Кызыл уул уулу Дөөлөс","bio":"-","url":"https://www.sanjyra.net/man/44277"},{"id":"69859","source":"44277","name":"Толуман","fullName":"Дөөлөс уулу Толуман","bio":"-","url":"https://www.sanjyra.net/man/69859"},{"id":"69860","source":"69859","name":"Чулуман","fullName":"Толуман уулу Чулуман","bio":"-","url":"https://www.sanjyra.net/man/69860"},{"id":"69861","source":"69860","name":"Зумбул","fullName":"Чулуман уулу Зумбул","bio":"-","url":"https://www.sanjyra.net/man/69861"},{"id":"69862","source":"69861","name":"Мендеке","fullName":"Зумбул уулу Мендеке","bio":"-","url":"https://www.sanjyra.net/man/69862"},{"id":"69863","source":"69862","name":"Жаный","fullName":"Мендеке уулу Жаный","bio":"-","url":"https://www.sanjyra.net/man/69863"},{"id":"69864","source":"69863","name":"Кудаш","fullName":"Жаный уулу Кудаш","bio":"-","url":"https://www.sanjyra.net/man/69864"},{"id":"69865","source":"69864","name":"Байболот","fullName":"Кудаш уулу Байболот","bio":"-","url":"https://www.sanjyra.net/man/69865"},{"id":"69866","source":"69865","name":"Доскулу","fullName":"Байболот уулу Доскулу","bio":"-","url":"https://www.sanjyra.net/man/69866"},{"id":"69867","source":"69866","name":"Сарт аке","fullName":"Доскулу уулу Сарт аке","bio":"-","url":"https://www.sanjyra.net/man/69867"},{"id":"71518","source":"69867","name":"Улак","fullName":"Сарт аке уулу Улак","bio":"-","url":"https://www.sanjyra.net/man/71518"},{"id":"71519","source":"71518","name":"Өмүрзак","fullName":"Улак уулу Өмүрзак","bio":"-","url":"https://www.sanjyra.net/man/71519"},{"id":"71520","source":"71519","name":"Күгүнбай","fullName":"Өмүрзак уулу Күгүнбай","bio":"-","url":"https://www.sanjyra.net/man/71520"},{"id":"71521","source":"71520","name":"Имай","fullName":"Күгүнбай уулу Имай","bio":"-","url":"https://www.sanjyra.net/man/71521"},{"id":"71522","source":"71521","name":"Кулубек","fullName":"Имай уулу Кулубек","bio":"-","url":"https://www.sanjyra.net/man/71522"},{"id":"71523","source":"71522","name":"Касен","fullName":"Кулубек уулу Касен","bio":"-","url":"https://www.sanjyra.net/man/71523"},{"id":"71524","source":"71523","name":"Бакыт","fullName":"Касен уулу Бакыт","bio":"-","url":"https://www.sanjyra.net/man/71524"},{"id":"71526","source":"71524","name":"Даниэль","fullName":"Бакыт уулу Даниэль","bio":"-","url":"https://www.sanjyra.net/man/71526"},{"id":"71525","source":"71523","name":"Бактияр","fullName":"Касен уулу Бактияр","bio":"-","url":"https://www.sanjyra.net/man/71525"},{"id":"44278","source":"44275","name":"Тейит","fullName":"Кызыл уул уулу Тейит","bio":"-","url":"https://www.sanjyra.net/man/44278"},{"id":"46635","source":"44278","name":"Арыктейит","fullName":"Тейит уулу Арыктейит","bio":"-","url":"https://www.sanjyra.net/man/46635"},{"id":"46636","source":"44278","name":"Каратейит","fullName":"Тейит уулу Каратейит","bio":"-","url":"https://www.sanjyra.net/man/46636"},{"id":"46637","source":"44278","name":"Сарытейит","fullName":"Тейит уулу Сарытейит","bio":"-","url":"https://www.sanjyra.net/man/46637"},{"id":"46641","source":"46637","name":"Айдар","fullName":"Сарытейит уулу Айдар","bio":"-","url":"https://www.sanjyra.net/man/46641"},{"id":"46644","source":"46641","name":"Бука","fullName":"Айдар уулу Бука","bio":"-","url":"https://www.sanjyra.net/man/46644"},{"id":"46645","source":"46641","name":"Бөрү","fullName":"Айдар уулу Бөрү","bio":"-","url":"https://www.sanjyra.net/man/46645"},{"id":"46646","source":"46641","name":"Жаманак","fullName":"Айдар уулу Жаманак","bio":"-","url":"https://www.sanjyra.net/man/46646"},{"id":"46647","source":"46641","name":"Кочкор","fullName":"Айдар уулу Кочкор","bio":"-","url":"https://www.sanjyra.net/man/46647"},{"id":"46648","source":"46641","name":"Отунчу","fullName":"Айдар уулу Отунчу","bio":"-","url":"https://www.sanjyra.net/man/46648"},{"id":"46642","source":"46637","name":"Мама","fullName":"Сарытейит уулу Мама","bio":"-","url":"https://www.sanjyra.net/man/46642"},{"id":"46649","source":"46642","name":"Зарип","fullName":"Мама уулу Зарип","bio":"-","url":"https://www.sanjyra.net/man/46649"},{"id":"46650","source":"46642","name":"Алапа","fullName":"Мама уулу Алапа","bio":"-","url":"https://www.sanjyra.net/man/46650"},{"id":"46651","source":"46642","name":"Дүрбөөл","fullName":"Мама уулу Дүрбөөл","bio":"-","url":"https://www.sanjyra.net/man/46651"},{"id":"46652","source":"46642","name":"Нооруз","fullName":"Мама уулу Нооруз","bio":"-","url":"https://www.sanjyra.net/man/46652"},{"id":"46653","source":"46642","name":"Шайым","fullName":"Мама уулу Шайым","bio":"-","url":"https://www.sanjyra.net/man/46653"},{"id":"46643","source":"46637","name":"Кутан","fullName":"Сарытейит уулу Кутан","bio":"-","url":"https://www.sanjyra.net/man/46643"},{"id":"46654","source":"46643","name":"Баркы","fullName":"Кутан уулу Баркы","bio":"-","url":"https://www.sanjyra.net/man/46654"},{"id":"46655","source":"46643","name":"Кызылбаш","fullName":"Кутан уулу Кызылбаш","bio":"-","url":"https://www.sanjyra.net/man/46655"},{"id":"46656","source":"46643","name":"Чегитир","fullName":"Кутан уулу Чегитир","bio":"-","url":"https://www.sanjyra.net/man/46656"},{"id":"46657","source":"46643","name":"Чогорок","fullName":"Кутан уулу Чогорок","bio":"-","url":"https://www.sanjyra.net/man/46657"},{"id":"46658","source":"46643","name":"Чой","fullName":"Кутан уулу Чой","bio":"-","url":"https://www.sanjyra.net/man/46658"},{"id":"46659","source":"46643","name":"Чуулдай","fullName":"Кутан уулу Чуулдай","bio":"-","url":"https://www.sanjyra.net/man/46659"},{"id":"46638","source":"44278","name":"Байтейит","fullName":"Тейит уулу Байтейит","bio":"-","url":"https://www.sanjyra.net/man/46638"},{"id":"46639","source":"44278","name":"Уйгуртейит","fullName":"Тейит уулу Уйгуртейит","bio":"-","url":"https://www.sanjyra.net/man/46639"},{"id":"46640","source":"44278","name":"Чалтейит","fullName":"Тейит уулу Чалтейит","bio":"-","url":"https://www.sanjyra.net/man/46640"},{"id":"44279","source":"44275","name":"Найман","fullName":"Кызыл уул уулу Найман","bio":"-","url":"https://www.sanjyra.net/man/44279"},{"id":"44280","source":"44275","name":"Кыпчак","fullName":"Кызыл уул уулу Кыпчак","bio":"-","url":"https://www.sanjyra.net/man/44280"},{"id":"44281","source":"44275","name":"Бостон","fullName":"Кызыл уул уулу Бостон","bio":"-","url":"https://www.sanjyra.net/man/44281"},{"id":"44282","source":"44275","name":"Нойгут","fullName":"Кызыл уул уулу Нойгут","bio":"-","url":"https://www.sanjyra.net/man/44282"},{"id":"44283","source":"44275","name":"Аваат","fullName":"Кызыл уул уулу Аваат","bio":"-","url":"https://www.sanjyra.net/man/44283"},{"id":"66545","source":"44283","name":"Адалакы","fullName":"Аваат уулу Адалакы","bio":"-","url":"https://www.sanjyra.net/man/66545"},{"id":"66546","source":"66545","name":"Көчөк баатыр","fullName":"Адалакы уулу Көчөк баатыр","bio":"-","url":"https://www.sanjyra.net/man/66546"},{"id":"66547","source":"66546","name":"Мирике","fullName":"Көчөк баатыр уулу Мирике","bio":"-","url":"https://www.sanjyra.net/man/66547"},{"id":"66548","source":"66547","name":"Ырыс","fullName":"Мирике уулу Ырыс","bio":"-","url":"https://www.sanjyra.net/man/66548"},{"id":"66549","source":"66548","name":"Айдар","fullName":"Ырыс уулу Айдар","bio":"-","url":"https://www.sanjyra.net/man/66549"},{"id":"66550","source":"66549","name":"Балтакул","fullName":"Айдар уулу Балтакул","bio":"-","url":"https://www.sanjyra.net/man/66550"},{"id":"66551","source":"66550","name":"Нармаат","fullName":"Балтакул уулу Нармаат","bio":"-","url":"https://www.sanjyra.net/man/66551"},{"id":"66552","source":"66551","name":"Абдураим","fullName":"Нармаат уулу Абдураим","bio":"-","url":"https://www.sanjyra.net/man/66552"},{"id":"66553","source":"66552","name":"Кулмахамат","fullName":"Абдураим уулу Кулмахамат","bio":"-","url":"https://www.sanjyra.net/man/66553"},{"id":"66557","source":"66553","name":"Амир","fullName":"Кулмахамат уулу Амир","bio":"-","url":"https://www.sanjyra.net/man/66557"},{"id":"66566","source":"66557","name":"Кыйматай","fullName":"Амир уулу Кыйматай","bio":"-","url":"https://www.sanjyra.net/man/66566"},{"id":"66567","source":"66557","name":"Өмүрбек","fullName":"Амир уулу Өмүрбек","bio":"-","url":"https://www.sanjyra.net/man/66567"},{"id":"66568","source":"66557","name":"Арзыбек","fullName":"Амир уулу Арзыбек","bio":"-","url":"https://www.sanjyra.net/man/66568"},{"id":"66558","source":"66553","name":"Раимберди","fullName":"Кулмахамат уулу Раимберди","bio":"-","url":"https://www.sanjyra.net/man/66558"},{"id":"66560","source":"66558","name":"Кубаныч","fullName":"Раимберди уулу Кубаныч","bio":"-","url":"https://www.sanjyra.net/man/66560"},{"id":"66561","source":"66558","name":"Кубат","fullName":"Раимберди уулу Кубат","bio":"-","url":"https://www.sanjyra.net/man/66561"},{"id":"66562","source":"66558","name":"Талант","fullName":"Раимберди уулу Талант","bio":"-","url":"https://www.sanjyra.net/man/66562"},{"id":"66563","source":"66558","name":"Медет","fullName":"Раимберди уулу Медет","bio":"-","url":"https://www.sanjyra.net/man/66563"},{"id":"66559","source":"66553","name":"Момун (Абдумалик)","fullName":"Кулмахамат уулу Момун (Абдумалик)","bio":"-","url":"https://www.sanjyra.net/man/66559"},{"id":"66564","source":"66559","name":"Акжол","fullName":"Момун (Абдумалик) уулу Акжол","bio":"-","url":"https://www.sanjyra.net/man/66564"},{"id":"66565","source":"66559","name":"Нурмахамат","fullName":"Момун (Абдумалик) уулу Нурмахамат","bio":"-","url":"https://www.sanjyra.net/man/66565"},{"id":"66554","source":"66553","name":"Жалил","fullName":"Кулмахамат уулу Жалил","bio":"-","url":"https://www.sanjyra.net/man/66554"},{"id":"66555","source":"66554","name":"Нурсултан","fullName":"Жалил уулу Нурсултан","bio":"-","url":"https://www.sanjyra.net/man/66555"},{"id":"66556","source":"66554","name":"Жолдубай","fullName":"Жалил уулу Жолдубай","bio":"-","url":"https://www.sanjyra.net/man/66556"},{"id":"66569","source":"66553","name":"Расулберди","fullName":"Кулмахамат уулу Расулберди","bio":"-","url":"https://www.sanjyra.net/man/66569"},{"id":"66570","source":"66569","name":"Данагул","fullName":"Расулберди уулу Данагул","bio":"-","url":"https://www.sanjyra.net/man/66570"},{"id":"66571","source":"66569","name":"Садагул","fullName":"Расулберди уулу Садагул","bio":"-","url":"https://www.sanjyra.net/man/66571"},{"id":"66572","source":"66569","name":"Нуртаза","fullName":"Расулберди уулу Нуртаза","bio":"-","url":"https://www.sanjyra.net/man/66572"},{"id":"66573","source":"66569","name":"Нургазы","fullName":"Расулберди уулу Нургазы","bio":"-","url":"https://www.sanjyra.net/man/66573"},{"id":"44284","source":"44275","name":"Кесек","fullName":"Кызыл уул уулу Кесек","bio":"-","url":"https://www.sanjyra.net/man/44284"}]