diff --git a/README.md b/README.md index 665d534..b19d497 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,3 @@ # store-scraper-bot Telegram bot that support scrape infos of an app on stores - -## Credits - -Many thanks to [facundoolano](https://github.com/facundoolano)' -s [google-play-scraper](https://github.com/facundoolano/google-play-scraper) -and [app-store-scraper](https://github.com/facundoolano/app-store-scraper), myy api code are based -on -their logics. diff --git a/lombok.config b/lombok.config index b1b08aa..9bb7a6e 100644 --- a/lombok.config +++ b/lombok.config @@ -1,4 +1,3 @@ -lombok.accessors.fluent = true lombok.addLombokGeneratedAnnotation = true lombok.experimental.flagUsage = error lombok.log.apacheCommons.flagUsage = error diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java b/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java index 8cdc5e7..cf04f33 100644 --- a/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java +++ b/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java @@ -1,15 +1,14 @@ package com.miti99.storescraperbot.api.apple; -import com.miti99.storescraperbot.api.apple.entity.AppleAppDetail; -import com.miti99.storescraperbot.api.apple.request.ITunesLookupRequest; -import com.miti99.storescraperbot.api.apple.response.ITunesLookupResponse; +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; +import com.miti99.storescraperbot.api.apple.response.AppleAppResponse; import com.miti99.storescraperbot.repository.AppleAppRepository; import com.miti99.storescraperbot.util.GsonUtil; -import com.miti99.storescraperbot.util.RequestUtil; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpClient.Redirect; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse.BodyHandlers; import java.time.Instant; import java.time.LocalDate; @@ -22,16 +21,16 @@ import lombok.extern.log4j.Log4j2; @Log4j2 @NoArgsConstructor(access = AccessLevel.PRIVATE) public class AppStoreScraper { - private static final String LOOKUP_URL = "https://itunes.apple.com/lookup?"; + public static final String BASE_URL = "https://store-scraper.vercel.app/apple/"; @SneakyThrows - public static String rawLookup(ITunesLookupRequest request) { + public static String rawApp(AppleAppRequest request) { var httpRequest = HttpRequest.newBuilder() - .uri(URI.create(LOOKUP_URL + RequestUtil.makeGetParams(request))) + .uri(URI.create(BASE_URL + "/app")) // .timeout(Duration.ofMillis(TIMEOUT)) .header("Content-Type", "application/json") - .GET() + .POST(BodyPublishers.ofString(GsonUtil.toJson(request))) .build(); try (var httpClient = @@ -41,55 +40,64 @@ public class AppStoreScraper { .build()) { return httpClient.send(httpRequest, BodyHandlers.ofString()).body(); } catch (Exception e) { - log.error("rawLookup error - request: '{}'", GsonUtil.toJson(request), e); + log.error("rawAppResponse error - request: '{}'", GsonUtil.toJson(request), e); return null; } } @SneakyThrows - public static AppleAppDetail app(ITunesLookupRequest request) { - return GsonUtil.fromJson(rawLookup(request), ITunesLookupResponse.class).getAppDetail(); + public static AppleAppResponse app(AppleAppRequest request) { + return GsonUtil.fromJson(rawApp(request), AppleAppResponse.class); } - public static AppleAppDetail app(String appId) { - return app(new ITunesLookupRequest(appId)); - } - - public static AppleAppDetail app(long id) { - return app(new ITunesLookupRequest(id)); - } - - public static AppleAppDetail getAppResponse(String appId) { + public static AppleAppResponse getAppResponse(String appId, String country) { boolean isInCache = AppleAppRepository.INSTANCE.exist(appId); if (isInCache) { var app = AppleAppRepository.INSTANCE.load(appId); - return app.detail(); + return app.getApp(); } else { - var response = app(appId); + var response = app(new AppleAppRequest(appId, country)); AppleAppRepository.INSTANCE.init(appId); var app = AppleAppRepository.INSTANCE.load(appId); - app.detail(response); + app.setApp(response); AppleAppRepository.INSTANCE.save(appId, app); return response; } } - public static LocalDate getAppUpdated(String appId) { - var detail = getAppResponse(appId); - if (detail == null) { - log.error("detail is null"); + public static LocalDate getAppUpdated(String appId, String country) { + var response = getAppResponse(appId, country); + if (response == null) { + log.error("response is null"); return LocalDate.ofInstant(Instant.ofEpochMilli(0), ZoneId.systemDefault()); } - return LocalDate.ofInstant( - Instant.parse(detail.currentVersionReleaseDate()), ZoneId.systemDefault()); + return LocalDate.ofInstant(Instant.parse(response.updated()), ZoneId.systemDefault()); } - public static double getAppScore(String appId) { - var response = getAppResponse(appId); + public static double getAppScore(String appId, String country) { + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return 0.0; } - return response.averageUserRating(); + return response.score(); + } + + public static long getAppReviews(String appId, String country) { + var response = getAppResponse(appId, country); + if (response == null) { + log.error("response is null"); + return 0L; + } + return response.reviews(); + } + + public static long getAppRatings(String appId, String country) { + var response = getAppResponse(appId, country); + if (response == null) { + log.error("response is null"); + return 0L; + } + return response.ratings(); } } diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/entity/AppleAppDetail.java b/src/main/java/com/miti99/storescraperbot/api/apple/entity/AppleAppDetail.java deleted file mode 100644 index 79760db..0000000 --- a/src/main/java/com/miti99/storescraperbot/api/apple/entity/AppleAppDetail.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.miti99.storescraperbot.api.apple.entity; - -import lombok.Value; - -/** - * Chỉ define một số field cần thiết nên không đầy đủ. Khi cần thì check raw response để define thêm - */ -@Value -public class AppleAppDetail { - String bundleId; - String currentVersionReleaseDate; - double averageUserRating; -} diff --git a/src/main/java/com/miti99/storescraperbot/api/old/apple/request/AppleAppRequest.java b/src/main/java/com/miti99/storescraperbot/api/apple/request/AppleAppRequest.java similarity index 82% rename from src/main/java/com/miti99/storescraperbot/api/old/apple/request/AppleAppRequest.java rename to src/main/java/com/miti99/storescraperbot/api/apple/request/AppleAppRequest.java index 864da54..19f5f22 100644 --- a/src/main/java/com/miti99/storescraperbot/api/old/apple/request/AppleAppRequest.java +++ b/src/main/java/com/miti99/storescraperbot/api/apple/request/AppleAppRequest.java @@ -1,4 +1,4 @@ -package com.miti99.storescraperbot.api.old.apple.request; +package com.miti99.storescraperbot.api.apple.request; public record AppleAppRequest(Long id, String appId, String country, Boolean ratings) { diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/request/ITunesLookupRequest.java b/src/main/java/com/miti99/storescraperbot/api/apple/request/ITunesLookupRequest.java deleted file mode 100644 index b8bb058..0000000 --- a/src/main/java/com/miti99/storescraperbot/api/apple/request/ITunesLookupRequest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.miti99.storescraperbot.api.apple.request; - -import lombok.RequiredArgsConstructor; -import lombok.Value; - -@RequiredArgsConstructor -@Value -public class ITunesLookupRequest { - Long id; - String bundleId; - String entity = "software"; - - public ITunesLookupRequest(Long id) { - this(id, null); - } - - public ITunesLookupRequest(String bundleId) { - this(null, bundleId); - } -} diff --git a/src/main/java/com/miti99/storescraperbot/api/old/apple/response/AppleAppResponse.java b/src/main/java/com/miti99/storescraperbot/api/apple/response/AppleAppResponse.java similarity index 93% rename from src/main/java/com/miti99/storescraperbot/api/old/apple/response/AppleAppResponse.java rename to src/main/java/com/miti99/storescraperbot/api/apple/response/AppleAppResponse.java index e3f5da6..535bfa8 100644 --- a/src/main/java/com/miti99/storescraperbot/api/old/apple/response/AppleAppResponse.java +++ b/src/main/java/com/miti99/storescraperbot/api/apple/response/AppleAppResponse.java @@ -1,4 +1,4 @@ -package com.miti99.storescraperbot.api.old.apple.response; +package com.miti99.storescraperbot.api.apple.response; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/response/ITunesLookupResponse.java b/src/main/java/com/miti99/storescraperbot/api/apple/response/ITunesLookupResponse.java deleted file mode 100644 index 9e19a14..0000000 --- a/src/main/java/com/miti99/storescraperbot/api/apple/response/ITunesLookupResponse.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.miti99.storescraperbot.api.apple.response; - -import com.miti99.storescraperbot.api.apple.entity.AppleAppDetail; -import com.miti99.storescraperbot.util.GsonUtil; -import java.util.List; -import lombok.Value; -import lombok.extern.log4j.Log4j2; - -@Log4j2 -@Value -public class ITunesLookupResponse { - int resultCount; - List results; - - public AppleAppDetail getAppDetail() { - if (resultCount != 1) { - log.warn("resultCount('{}') != 1", resultCount); - log.warn("results: {}", GsonUtil.toJson(results)); - } - - return results.getFirst(); - } -} diff --git a/src/main/java/com/miti99/storescraperbot/api/old/google/GooglePlayScraper.java b/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java similarity index 90% rename from src/main/java/com/miti99/storescraperbot/api/old/google/GooglePlayScraper.java rename to src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java index f0f9d1b..c33a268 100644 --- a/src/main/java/com/miti99/storescraperbot/api/old/google/GooglePlayScraper.java +++ b/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java @@ -1,7 +1,7 @@ -package com.miti99.storescraperbot.api.old.google; +package com.miti99.storescraperbot.api.google; -import com.miti99.storescraperbot.api.old.google.request.GoogleAppRequest; -import com.miti99.storescraperbot.api.old.google.response.GoogleAppResponse; +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.api.google.response.GoogleAppResponse; import com.miti99.storescraperbot.repository.GoogleAppRepository; import com.miti99.storescraperbot.util.GsonUtil; import java.net.URI; @@ -21,7 +21,7 @@ import lombok.extern.log4j.Log4j2; @Log4j2 @NoArgsConstructor(access = AccessLevel.PRIVATE) public class GooglePlayScraper { - public static final String BASE_URL = "https://miti-google-play-scraper.vercel.app/"; + public static final String BASE_URL = "https://store-scraper.vercel.app/google/"; @SneakyThrows public static String rawApp(GoogleAppRequest request) { @@ -53,12 +53,12 @@ public class GooglePlayScraper { boolean isInCache = GoogleAppRepository.INSTANCE.exist(appId); if (isInCache) { var app = GoogleAppRepository.INSTANCE.load(appId); - return app.app(); + return app.getApp(); } else { var response = app(new GoogleAppRequest(appId, country)); GoogleAppRepository.INSTANCE.init(appId); var app = GoogleAppRepository.INSTANCE.load(appId); - app.app(response); + app.setApp(response); GoogleAppRepository.INSTANCE.save(appId, app); return response; } diff --git a/src/main/java/com/miti99/storescraperbot/api/old/google/request/GoogleAppRequest.java b/src/main/java/com/miti99/storescraperbot/api/google/request/GoogleAppRequest.java similarity index 69% rename from src/main/java/com/miti99/storescraperbot/api/old/google/request/GoogleAppRequest.java rename to src/main/java/com/miti99/storescraperbot/api/google/request/GoogleAppRequest.java index 7637d99..4c2066a 100644 --- a/src/main/java/com/miti99/storescraperbot/api/old/google/request/GoogleAppRequest.java +++ b/src/main/java/com/miti99/storescraperbot/api/google/request/GoogleAppRequest.java @@ -1,4 +1,4 @@ -package com.miti99.storescraperbot.api.old.google.request; +package com.miti99.storescraperbot.api.google.request; public record GoogleAppRequest(String appId, String country) { public GoogleAppRequest(String appId) { diff --git a/src/main/java/com/miti99/storescraperbot/api/old/google/response/GoogleAppResponse.java b/src/main/java/com/miti99/storescraperbot/api/google/response/GoogleAppResponse.java similarity index 96% rename from src/main/java/com/miti99/storescraperbot/api/old/google/response/GoogleAppResponse.java rename to src/main/java/com/miti99/storescraperbot/api/google/response/GoogleAppResponse.java index 3ab801d..7e9c2ef 100644 --- a/src/main/java/com/miti99/storescraperbot/api/old/google/response/GoogleAppResponse.java +++ b/src/main/java/com/miti99/storescraperbot/api/google/response/GoogleAppResponse.java @@ -1,4 +1,4 @@ -package com.miti99.storescraperbot.api.old.google.response; +package com.miti99.storescraperbot.api.google.response; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/miti99/storescraperbot/api/old/apple/AppStoreScraper.java b/src/main/java/com/miti99/storescraperbot/api/old/apple/AppStoreScraper.java deleted file mode 100644 index 35a4370..0000000 --- a/src/main/java/com/miti99/storescraperbot/api/old/apple/AppStoreScraper.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.miti99.storescraperbot.api.old.apple; - -import com.miti99.storescraperbot.api.old.apple.request.AppleAppRequest; -import com.miti99.storescraperbot.api.old.apple.response.AppleAppResponse; -import com.miti99.storescraperbot.repository.AppleAppRepository; -import com.miti99.storescraperbot.util.GsonUtil; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpClient.Redirect; -import java.net.http.HttpRequest; -import java.net.http.HttpRequest.BodyPublishers; -import java.net.http.HttpResponse.BodyHandlers; -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneId; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.log4j.Log4j2; - -@Log4j2 -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class AppStoreScraper { - public static final String BASE_URL = "https://miti-app-store-scraper.vercel.app/"; - - @SneakyThrows - public static String rawApp(AppleAppRequest request) { - var httpRequest = - HttpRequest.newBuilder() - .uri(URI.create(BASE_URL + "/app")) - // .timeout(Duration.ofMillis(TIMEOUT)) - .header("Content-Type", "application/json") - .POST(BodyPublishers.ofString(GsonUtil.toJson(request))) - .build(); - - try (var httpClient = - HttpClient.newBuilder() - .followRedirects(Redirect.NORMAL) - // .connectTimeout(Duration.ofMillis(TIMEOUT)) - .build()) { - return httpClient.send(httpRequest, BodyHandlers.ofString()).body(); - } catch (Exception e) { - log.error("rawAppResponse error - request: '{}'", GsonUtil.toJson(request), e); - return null; - } - } - - @SneakyThrows - public static AppleAppResponse app(AppleAppRequest request) { - return GsonUtil.fromJson(rawApp(request), AppleAppResponse.class); - } - - public static AppleAppResponse getAppResponse(String appId, String country) { - boolean isInCache = AppleAppRepository.INSTANCE.exist(appId); - if (isInCache) { - var app = AppleAppRepository.INSTANCE.load(appId); - return app.app(); - } else { - var response = app(new AppleAppRequest(appId, country)); - AppleAppRepository.INSTANCE.init(appId); - var app = AppleAppRepository.INSTANCE.load(appId); - app.app(response); - AppleAppRepository.INSTANCE.save(appId, app); - return response; - } - } - - public static LocalDate getAppUpdated(String appId, String country) { - var response = getAppResponse(appId, country); - if (response == null) { - log.error("response is null"); - return LocalDate.ofInstant(Instant.ofEpochMilli(0), ZoneId.systemDefault()); - } - return LocalDate.ofInstant(Instant.parse(response.updated()), ZoneId.systemDefault()); - } - - public static double getAppScore(String appId, String country) { - var response = getAppResponse(appId, country); - if (response == null) { - log.error("response is null"); - return 0.0; - } - return response.score(); - } - - public static long getAppReviews(String appId, String country) { - var response = getAppResponse(appId, country); - if (response == null) { - log.error("response is null"); - return 0L; - } - return response.reviews(); - } - - public static long getAppRatings(String appId, String country) { - var response = getAppResponse(appId, country); - if (response == null) { - log.error("response is null"); - return 0L; - } - return response.ratings(); - } -} diff --git a/src/main/java/com/miti99/storescraperbot/api/old/package-info.java b/src/main/java/com/miti99/storescraperbot/api/old/package-info.java deleted file mode 100644 index 101cc5c..0000000 --- a/src/main/java/com/miti99/storescraperbot/api/old/package-info.java +++ /dev/null @@ -1,2 +0,0 @@ -@Deprecated -package com.miti99.storescraperbot.api.old; diff --git a/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java b/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java index 4b5d2cd..34d5dfa 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java +++ b/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java @@ -3,8 +3,8 @@ package com.miti99.storescraperbot.bot; import static com.miti99.storescraperbot.constant.Constant.VIETNAM_ZONE_ID; import static com.miti99.storescraperbot.constant.Constant.WEEKENDS; -import com.miti99.storescraperbot.api.old.apple.AppStoreScraper; -import com.miti99.storescraperbot.api.old.google.GooglePlayScraper; +import com.miti99.storescraperbot.api.apple.AppStoreScraper; +import com.miti99.storescraperbot.api.google.GooglePlayScraper; import com.miti99.storescraperbot.bot.command.AddAppleAppCommand; import com.miti99.storescraperbot.bot.command.AddGoogleAppCommand; import com.miti99.storescraperbot.bot.command.AddGroupCommand; @@ -79,7 +79,7 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot { public void runCheckApp() { var admin = AdminRepository.INSTANCE.load(); - for (var groupId : admin.groups()) { + for (var groupId : admin.getGroups()) { checkAppForGroup(groupId); } } @@ -89,7 +89,7 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot { var now = LocalDate.now(); var nonUpdatedAppleApps = new ArrayList(); - for (var app : group.appleApps()) { + for (var app : group.getAppleApps()) { var appId = app.appId(); var updated = AppStoreScraper.getAppUpdated(appId, app.country()); long days = ChronoUnit.DAYS.between(updated, now); @@ -98,7 +98,7 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot { } } var nonUpdatedGoogleApps = new ArrayList(); - for (var app : group.googleApps()) { + for (var app : group.getGoogleApps()) { var appId = app.appId(); var updated = GooglePlayScraper.getLastUpdateOfApp(appId, app.country()); long days = ChronoUnit.DAYS.between(updated, now); diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/AddAppleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/AddAppleAppCommand.java index 309c64d..9024fcf 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/AddAppleAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/AddAppleAppCommand.java @@ -1,8 +1,8 @@ package com.miti99.storescraperbot.bot.command; -import com.miti99.storescraperbot.api.old.apple.AppStoreScraper; -import com.miti99.storescraperbot.api.old.apple.request.AppleAppRequest; -import com.miti99.storescraperbot.api.old.apple.response.AppleAppResponse; +import com.miti99.storescraperbot.api.apple.AppStoreScraper; +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; +import com.miti99.storescraperbot.api.apple.response.AppleAppResponse; import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; import com.miti99.storescraperbot.model.entity.AppleAppInfo; import com.miti99.storescraperbot.repository.AdminRepository; @@ -26,7 +26,7 @@ public class AddAppleAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -64,13 +64,13 @@ public class AddAppleAppCommand extends BaseStoreScraperBotCommand { var group = GroupRepository.INSTANCE.load(groupId); var finalAppId = appId; - if (group.appleApps().stream().anyMatch(app -> finalAppId.equals(app.appId()))) { + if (group.getAppleApps().stream().anyMatch(app -> finalAppId.equals(app.appId()))) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Apple app %s is already added".formatted(appId)); return; } - group.appleApps().add(new AppleAppInfo(appId, country)); + group.getAppleApps().add(new AppleAppInfo(appId, country)); GroupRepository.INSTANCE.save(groupId, group); StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/AddGoogleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/AddGoogleAppCommand.java index c2a1eb8..cffd917 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/AddGoogleAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/AddGoogleAppCommand.java @@ -1,8 +1,8 @@ package com.miti99.storescraperbot.bot.command; -import com.miti99.storescraperbot.api.old.google.GooglePlayScraper; -import com.miti99.storescraperbot.api.old.google.request.GoogleAppRequest; -import com.miti99.storescraperbot.api.old.google.response.GoogleAppResponse; +import com.miti99.storescraperbot.api.google.GooglePlayScraper; +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.api.google.response.GoogleAppResponse; import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; import com.miti99.storescraperbot.model.entity.GoogleAppInfo; import com.miti99.storescraperbot.repository.AdminRepository; @@ -26,7 +26,7 @@ public class AddGoogleAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -52,13 +52,13 @@ public class AddGoogleAppCommand extends BaseStoreScraperBotCommand { long groupId = chat.getId(); var group = GroupRepository.INSTANCE.load(groupId); - if (group.googleApps().stream().anyMatch(app -> appId.equals(app.appId()))) { + if (group.getGoogleApps().stream().anyMatch(app -> appId.equals(app.appId()))) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Google app %s is already added".formatted(appId)); return; } - group.googleApps().add(new GoogleAppInfo(appId, country)); + group.getGoogleApps().add(new GoogleAppInfo(appId, country)); GroupRepository.INSTANCE.save(groupId, group); StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/AddGroupCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/AddGroupCommand.java index 776c67a..2e61d74 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/AddGroupCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/AddGroupCommand.java @@ -41,13 +41,13 @@ public class AddGroupCommand extends BaseStoreScraperBotCommand { } var admin = AdminRepository.INSTANCE.load(); - if (admin.groups().contains(groupId)) { + if (admin.getGroups().contains(groupId)) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group is already added"); return; } GroupRepository.INSTANCE.init(groupId); - admin.groups().add(groupId); + admin.getGroups().add(groupId); AdminRepository.INSTANCE.save(admin); StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group added successfully"); } diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppCommand.java index 258c02c..594b6a0 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppCommand.java @@ -1,7 +1,7 @@ package com.miti99.storescraperbot.bot.command; -import com.miti99.storescraperbot.api.old.apple.AppStoreScraper; -import com.miti99.storescraperbot.api.old.google.GooglePlayScraper; +import com.miti99.storescraperbot.api.apple.AppStoreScraper; +import com.miti99.storescraperbot.api.google.GooglePlayScraper; import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; import com.miti99.storescraperbot.bot.table.Table; import com.miti99.storescraperbot.constant.Constant; @@ -24,7 +24,7 @@ public class CheckAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -43,7 +43,7 @@ public class CheckAppCommand extends BaseStoreScraperBotCommand { sb.append("Apple Apps:\n"); sb.append("\n"); var appleTable = new Table("AppId", "Updated", "Days", "OK"); - for (var app : group.appleApps()) { + for (var app : group.getAppleApps()) { var appId = app.appId(); var updated = AppStoreScraper.getAppUpdated(appId, app.country()); long days = ChronoUnit.DAYS.between(updated, now); @@ -57,7 +57,7 @@ public class CheckAppCommand extends BaseStoreScraperBotCommand { sb.append("Google Apps:\n"); sb.append("\n"); var googleTable = new Table("AppId", "Updated", "Days", "OK"); - for (var app : group.googleApps()) { + for (var app : group.getGoogleApps()) { var appId = app.appId(); var updated = GooglePlayScraper.getLastUpdateOfApp(appId, app.country()); long days = ChronoUnit.DAYS.between(updated, now); diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppScoreCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppScoreCommand.java index 2a24c92..37b40d6 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppScoreCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/CheckAppScoreCommand.java @@ -1,7 +1,7 @@ package com.miti99.storescraperbot.bot.command; -import com.miti99.storescraperbot.api.old.apple.AppStoreScraper; -import com.miti99.storescraperbot.api.old.google.GooglePlayScraper; +import com.miti99.storescraperbot.api.apple.AppStoreScraper; +import com.miti99.storescraperbot.api.google.GooglePlayScraper; import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; import com.miti99.storescraperbot.bot.table.Table; import com.miti99.storescraperbot.repository.AdminRepository; @@ -23,7 +23,7 @@ public class CheckAppScoreCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -41,7 +41,7 @@ public class CheckAppScoreCommand extends BaseStoreScraperBotCommand { sb.append("Apple Apps:\n"); sb.append("\n"); var appleTable = new Table("AppId", "Score", "Ratings"); - for (var app : group.appleApps()) { + for (var app : group.getAppleApps()) { var appId = app.appId(); var country = app.country(); double score = @@ -55,7 +55,7 @@ public class CheckAppScoreCommand extends BaseStoreScraperBotCommand { sb.append("Google Apps:\n"); sb.append("\n"); var googleTable = new Table("AppId", "Score", "Ratings"); - for (var app : group.googleApps()) { + for (var app : group.getGoogleApps()) { var appId = app.appId(); var country = app.country(); double score = diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/DeleteAppleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/DeleteAppleAppCommand.java index 9865540..baa9d1f 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/DeleteAppleAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/DeleteAppleAppCommand.java @@ -18,7 +18,7 @@ public class DeleteAppleAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -33,12 +33,12 @@ public class DeleteAppleAppCommand extends BaseStoreScraperBotCommand { long groupId = chat.getId(); var group = GroupRepository.INSTANCE.load(groupId); - if (group.appleApps().stream().noneMatch(app -> appId.equals(app.appId()))) { + if (group.getAppleApps().stream().noneMatch(app -> appId.equals(app.appId()))) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Apple app is not added"); return; } - group.appleApps().removeIf(app -> appId.equals(app.appId())); + group.getAppleApps().removeIf(app -> appId.equals(app.appId())); GroupRepository.INSTANCE.save(groupId, group); StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Apple app deleted successfully"); diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGoogleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGoogleAppCommand.java index ce388e7..912e774 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGoogleAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGoogleAppCommand.java @@ -18,7 +18,7 @@ public class DeleteGoogleAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -33,12 +33,12 @@ public class DeleteGoogleAppCommand extends BaseStoreScraperBotCommand { long groupId = chat.getId(); var group = GroupRepository.INSTANCE.load(groupId); - if (group.googleApps().stream().noneMatch(app -> appId.equals(app.appId()))) { + if (group.getGoogleApps().stream().noneMatch(app -> appId.equals(app.appId()))) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Google app is not added"); return; } - group.googleApps().removeIf(app -> appId.equals(app.appId())); + group.getGoogleApps().removeIf(app -> appId.equals(app.appId())); GroupRepository.INSTANCE.save(groupId, group); StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Google app deleted successfully"); diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGroupCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGroupCommand.java index 5cc5400..5cfd9a6 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGroupCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/DeleteGroupCommand.java @@ -40,12 +40,12 @@ public class DeleteGroupCommand extends BaseStoreScraperBotCommand { } var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(groupId)) { + if (!admin.getGroups().contains(groupId)) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group is not added"); return; } - admin.groups().remove(groupId); + admin.getGroups().remove(groupId); AdminRepository.INSTANCE.save(admin); StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group deleted successfully"); } diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/ListAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/ListAppCommand.java index 896faf5..138fc77 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/ListAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/ListAppCommand.java @@ -19,7 +19,7 @@ public class ListAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; @@ -38,7 +38,7 @@ public class ListAppCommand extends BaseStoreScraperBotCommand { sb.append("\n"); var appleTable = new Table("#", "AppId", "Country"); int i = 0; - for (var app : group.appleApps()) { + for (var app : group.getAppleApps()) { i++; appleTable.addRow(i, app.appId(), app.country()); } @@ -49,7 +49,7 @@ public class ListAppCommand extends BaseStoreScraperBotCommand { sb.append("\n"); var googleTable = new Table("#", "AppId", "Country"); i = 0; - for (var app : group.googleApps()) { + for (var app : group.getGoogleApps()) { i++; googleTable.addRow(i, app.appId(), app.country()); } diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/ListGroupCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/ListGroupCommand.java index 5c837d7..bf1fe52 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/ListGroupCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/ListGroupCommand.java @@ -28,7 +28,7 @@ public class ListGroupCommand extends BaseStoreScraperBotCommand { } var admin = AdminRepository.INSTANCE.load(); - var groups = admin.groups(); + var groups = admin.getGroups(); var sb = new StringBuilder(); sb.append("Groups:\n"); for (var groupId : groups) { diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java index fe1b804..b6c46ff 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java @@ -1,9 +1,10 @@ package com.miti99.storescraperbot.bot.command; -import com.miti99.storescraperbot.api.old.apple.AppStoreScraper; -import com.miti99.storescraperbot.api.old.apple.request.AppleAppRequest; +import com.miti99.storescraperbot.api.apple.AppStoreScraper; +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; import com.miti99.storescraperbot.repository.AdminRepository; +import com.miti99.storescraperbot.util.GsonUtil; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import lombok.SneakyThrows; @@ -29,7 +30,7 @@ public class RawAppleAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java index 2ac9044..59809b9 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java +++ b/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java @@ -1,9 +1,10 @@ package com.miti99.storescraperbot.bot.command; -import com.miti99.storescraperbot.api.old.google.GooglePlayScraper; -import com.miti99.storescraperbot.api.old.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.api.google.GooglePlayScraper; +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; import com.miti99.storescraperbot.repository.AdminRepository; +import com.miti99.storescraperbot.util.GsonUtil; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import lombok.SneakyThrows; @@ -29,7 +30,7 @@ public class RawGoogleAppCommand extends BaseStoreScraperBotCommand { protected void executeCommand( TelegramClient telegramClient, User user, Chat chat, String[] arguments) { var admin = AdminRepository.INSTANCE.load(); - if (!admin.groups().contains(chat.getId())) { + if (!admin.getGroups().contains(chat.getId())) { StoreScrapeBotTelegramClient.INSTANCE.sendMessage( chat.getId(), "Group is not allowed to use bot"); return; diff --git a/src/main/java/com/miti99/storescraperbot/model/AppleApp.java b/src/main/java/com/miti99/storescraperbot/model/AppleApp.java index d542224..4b1cbaa 100644 --- a/src/main/java/com/miti99/storescraperbot/model/AppleApp.java +++ b/src/main/java/com/miti99/storescraperbot/model/AppleApp.java @@ -1,7 +1,6 @@ package com.miti99.storescraperbot.model; -import com.miti99.storescraperbot.api.apple.entity.AppleAppDetail; -import com.miti99.storescraperbot.api.old.apple.response.AppleAppResponse; +import com.miti99.storescraperbot.api.apple.response.AppleAppResponse; import lombok.Getter; import lombok.Setter; @@ -9,5 +8,4 @@ import lombok.Setter; @Setter public class AppleApp extends AbstractModel { AppleAppResponse app; - AppleAppDetail detail; } diff --git a/src/main/java/com/miti99/storescraperbot/model/GoogleApp.java b/src/main/java/com/miti99/storescraperbot/model/GoogleApp.java index 5d43b61..de9b993 100644 --- a/src/main/java/com/miti99/storescraperbot/model/GoogleApp.java +++ b/src/main/java/com/miti99/storescraperbot/model/GoogleApp.java @@ -1,6 +1,6 @@ package com.miti99.storescraperbot.model; -import com.miti99.storescraperbot.api.old.google.response.GoogleAppResponse; +import com.miti99.storescraperbot.api.google.response.GoogleAppResponse; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java index 14a367d..330c3e1 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java @@ -71,7 +71,7 @@ public abstract class AbstractRepository> { return; } V data = classV.getDeclaredConstructor().newInstance(); - data.key(key); + data.setKey(key); save(key, data); } catch (Exception e) { log.error("Error while initializing data", e); diff --git a/src/main/java/com/miti99/storescraperbot/util/GooglePlayAppScraper.java b/src/main/java/com/miti99/storescraperbot/util/GooglePlayAppScraper.java deleted file mode 100644 index ab599b7..0000000 --- a/src/main/java/com/miti99/storescraperbot/util/GooglePlayAppScraper.java +++ /dev/null @@ -1,654 +0,0 @@ -package com.miti99.storescraperbot.util; - -import java.io.*; -import java.net.*; -import java.util.*; -import java.util.regex.*; -import com.google.gson.*; - -public class GooglePlayAppScraper { - - private static final String BASE_URL = "https://play.google.com"; - private static final String PLAYSTORE_URL = BASE_URL + "/store/apps/details"; - - /** - * Main method to fetch app details by appId - * @param appId The application ID (package name) to fetch details for - * @return JsonObject containing app details - * @throws Exception if there's an error during the request or parsing - */ - public static JsonObject getAppDetails(String appId) throws Exception { - return getAppDetails(appId, "en", "us"); - } - - /** - * Fetch app details by appId with language and country parameters - * @param appId The application ID (package name) to fetch details for - * @param lang Language code (e.g., "en") - * @param country Country code (e.g., "us") - * @return JsonObject containing app details - * @throws Exception if there's an error during the request or parsing - */ - public static JsonObject getAppDetails(String appId, String lang, String country) - throws Exception { - if (appId == null || appId.isEmpty()) { - throw new IllegalArgumentException("appId missing"); - } - - // Set default values for lang and country if not provided - if (lang == null || lang.isEmpty()) lang = "en"; - if (country == null || country.isEmpty()) country = "us"; - - // Construct URL with query parameters - // String url = String.format("%s?id=%s&hl=%s&gl=%s", PLAYSTORE_URL, appId, lang, country); - String url = String.format("%s?id=%s", PLAYSTORE_URL, appId); - - // Make HTTP request - String response = makeRequest(url); - - // Parse the response and extract data - JsonObject result = parseResponse(response); - - // Add appId and URL to the result - result.addProperty("appId", appId); - result.addProperty("url", url); - - return result; - } - - /** - * Make HTTP request to the Google Play Store - * @param url The URL to request - * @return The response as a String - * @throws Exception if there's an error during the request - */ - private static String makeRequest(String url) throws Exception { - URL obj = new URL(url); - HttpURLConnection con = (HttpURLConnection) obj.openConnection(); - con.setRequestMethod("GET"); - con.setRequestProperty( - "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"); - - int responseCode = con.getResponseCode(); - if (responseCode != 200) { - throw new RuntimeException("HTTP request failed with code: " + responseCode); - } - - BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream())); - String inputLine; - StringBuilder response = new StringBuilder(); - - while ((inputLine = in.readLine()) != null) { - response.append(inputLine); - } - in.close(); - - return response.toString(); - } - - /** - * Parse the Google Play Store response and extract app data - * @param response The HTML response from Google Play Store - * @return JsonObject with extracted app data - */ - private static JsonObject parseResponse(String response) { - JsonObject result = new JsonObject(); - - try { - // Extract script data using regex (similar to scriptData.js) - Map parsedData = parseScriptData(response); - - // Extract fields using mappings (similar to MAPPINGS in app.js) - extractFields(result, parsedData); - - } catch (Exception e) { - e.printStackTrace(); - } - - return result; - } - - /** - * Parse JavaScript objects from AF_initDataCallback functions - * @param response The HTML response - * @return Map of parsed data - */ - private static Map parseScriptData(String response) { - Map parsedData = new HashMap<>(); - - // Regex patterns to match script data - Pattern scriptPattern = - Pattern.compile(">AF_initDataCallback[\\s\\S]*? parsedData) { - // Extract all fields (full versions of the MAPPINGS from app.js) - try { - // Get the main data array - JsonElement ds5 = (JsonElement) parsedData.get("ds:5"); - - // Title - ['ds:5', 1, 2, 0, 0] - Object titleObj = getPath(ds5, 1, 2, 0, 0); - if (titleObj instanceof JsonElement && ((JsonElement) titleObj).isJsonPrimitive()) { - result.addProperty("title", ((JsonElement) titleObj).getAsString()); - } - - // Description - ['ds:5', 1, 2] with helper function - Object descriptionObj = getPath(ds5, 1, 2); - if (descriptionObj instanceof JsonElement) { - String description = extractDescriptionText((JsonElement) descriptionObj); - result.addProperty("description", description); - } - - // Description HTML - ['ds:5', 1, 2] with helper function - Object descriptionHtmlObj = getPath(ds5, 1, 2); - if (descriptionHtmlObj instanceof JsonElement) { - String descriptionHtml = extractDescriptionHtmlLocalized((JsonElement) descriptionHtmlObj); - result.addProperty("descriptionHTML", descriptionHtml); - } - - // Summary - ['ds:5', 1, 2, 73, 0, 1] - Object summaryObj = getPath(ds5, 1, 2, 73, 0, 1); - if (summaryObj instanceof JsonElement && ((JsonElement) summaryObj).isJsonPrimitive()) { - result.addProperty("summary", ((JsonElement) summaryObj).getAsString()); - } - - // Installs - ['ds:5', 1, 2, 13, 0] - Object installsObj = getPath(ds5, 1, 2, 13, 0); - if (installsObj instanceof JsonElement && ((JsonElement) installsObj).isJsonPrimitive()) { - result.addProperty("installs", ((JsonElement) installsObj).getAsString()); - } - - // Min Installs - ['ds:5', 1, 2, 13, 1] - Object minInstallsObj = getPath(ds5, 1, 2, 13, 1); - if (minInstallsObj instanceof JsonElement - && ((JsonElement) minInstallsObj).isJsonPrimitive()) { - result.addProperty("minInstalls", ((JsonElement) minInstallsObj).getAsLong()); - } - - // Max Installs - ['ds:5', 1, 2, 13, 2] - Object maxInstallsObj = getPath(ds5, 1, 2, 13, 2); - if (maxInstallsObj instanceof JsonElement - && ((JsonElement) maxInstallsObj).isJsonPrimitive()) { - result.addProperty("maxInstalls", ((JsonElement) maxInstallsObj).getAsLong()); - } - - // Score - ['ds:5', 1, 2, 51, 0, 1] - Object scoreObj = getPath(ds5, 1, 2, 51, 0, 1); - if (scoreObj instanceof JsonElement && ((JsonElement) scoreObj).isJsonPrimitive()) { - result.addProperty("score", ((JsonElement) scoreObj).getAsFloat()); - } - - // Score Text - ['ds:5', 1, 2, 51, 0, 0] - Object scoreTextObj = getPath(ds5, 1, 2, 51, 0, 0); - if (scoreTextObj instanceof JsonElement && ((JsonElement) scoreTextObj).isJsonPrimitive()) { - result.addProperty("scoreText", ((JsonElement) scoreTextObj).getAsString()); - } - - // Ratings - ['ds:5', 1, 2, 51, 2, 1] - Object ratingsObj = getPath(ds5, 1, 2, 51, 2, 1); - if (ratingsObj instanceof JsonElement && ((JsonElement) ratingsObj).isJsonPrimitive()) { - result.addProperty("ratings", ((JsonElement) ratingsObj).getAsLong()); - } - - // Reviews - ['ds:5', 1, 2, 51, 3, 1] - Object reviewsObj = getPath(ds5, 1, 2, 51, 3, 1); - if (reviewsObj instanceof JsonElement && ((JsonElement) reviewsObj).isJsonPrimitive()) { - result.addProperty("reviews", ((JsonElement) reviewsObj).getAsLong()); - } - - // Histogram - ['ds:5', 1, 2, 51, 1] with helper function - Object histogramObj = getPath(ds5, 1, 2, 51, 1); - JsonObject histogram = buildHistogram(histogramObj); - result.add("histogram", histogram); - - // Price - ['ds:5', 1, 2, 57, 0, 0, 0, 0, 1, 0, 0] with function - Object priceObj = getPath(ds5, 1, 2, 57, 0, 0, 0, 0, 1, 0, 0); - double price = 0; - if (priceObj instanceof JsonElement && ((JsonElement) priceObj).isJsonPrimitive()) { - price = ((JsonElement) priceObj).getAsLong() / 1000000.0; - } - result.addProperty("price", price); - - // Original Price - ['ds:5', 1, 2, 57, 0, 0, 0, 0, 1, 1, 0] with function - Object originalPriceObj = getPath(ds5, 1, 2, 57, 0, 0, 0, 0, 1, 1, 0); - if (originalPriceObj instanceof JsonElement - && ((JsonElement) originalPriceObj).isJsonPrimitive()) { - long originalPriceValue = ((JsonElement) originalPriceObj).getAsLong(); - if (originalPriceValue > 0) { - result.addProperty("originalPrice", originalPriceValue / 1000000.0); - } - } - - // Discount End Date - ['ds:5', 1, 2, 57, 0, 0, 0, 0, 14, 1] - Object discountEndDateObj = getPath(ds5, 1, 2, 57, 0, 0, 0, 0, 14, 1); - if (discountEndDateObj instanceof JsonElement - && ((JsonElement) discountEndDateObj).isJsonPrimitive()) { - result.addProperty("discountEndDate", ((JsonElement) discountEndDateObj).getAsLong()); - } - - // Free - ['ds:5', 1, 2, 57, 0, 0, 0, 0, 1, 0, 0] with function - boolean isFree = false; - if (priceObj instanceof JsonElement && ((JsonElement) priceObj).isJsonPrimitive()) { - isFree = ((JsonElement) priceObj).getAsLong() == 0; - } - result.addProperty("free", isFree); - - // Currency - ['ds:5', 1, 2, 57, 0, 0, 0, 0, 1, 0, 1] - Object currencyObj = getPath(ds5, 1, 2, 57, 0, 0, 0, 0, 1, 0, 1); - if (currencyObj instanceof JsonElement && ((JsonElement) currencyObj).isJsonPrimitive()) { - result.addProperty("currency", ((JsonElement) currencyObj).getAsString()); - } - - // Price Text - ['ds:5', 1, 2, 57, 0, 0, 0, 0, 1, 0, 2] with helper - Object priceTextPathObj = getPath(ds5, 1, 2, 57, 0, 0, 0, 0, 1, 0, 2); - String priceText = "Free"; - if (priceTextPathObj instanceof JsonElement - && ((JsonElement) priceTextPathObj).isJsonPrimitive()) { - priceText = ((JsonElement) priceTextPathObj).getAsString(); - } - result.addProperty("priceText", priceText); - - // Available - ['ds:5', 1, 2, 18, 0] with Boolean function - Object availableObj = getPath(ds5, 1, 2, 18, 0); - boolean isAvailable = false; - if (availableObj instanceof JsonElement && ((JsonElement) availableObj).isJsonPrimitive()) { - isAvailable = ((JsonElement) availableObj).getAsInt() == 1; - } - result.addProperty("available", isAvailable); - - // Offers IAP - ['ds:5', 1, 2, 19, 0] with Boolean function - Object offersIAPObj = getPath(ds5, 1, 2, 19, 0); - boolean offersIAP = false; - if (offersIAPObj instanceof JsonElement && ((JsonElement) offersIAPObj).isJsonPrimitive()) { - offersIAP = ((JsonElement) offersIAPObj).getAsInt() == 1; - } - result.addProperty("offersIAP", offersIAP); - - // IAP Range - ['ds:5', 1, 2, 19, 0] - Object iapRangeObj = getPath(ds5, 1, 2, 19, 0); - if (iapRangeObj instanceof JsonElement && ((JsonElement) iapRangeObj).isJsonPrimitive()) { - result.addProperty("IAPRange", ((JsonElement) iapRangeObj).getAsString()); - } - - // Android Version - ['ds:5', 1, 2, 140, 1, 1, 0, 0, 1] with helper - Object androidVersionObj = getPath(ds5, 1, 2, 140, 1, 1, 0, 0, 1); - String androidVersion = "VARY"; - if (androidVersionObj instanceof JsonElement - && ((JsonElement) androidVersionObj).isJsonPrimitive()) { - androidVersion = normalizeAndroidVersion(((JsonElement) androidVersionObj).getAsString()); - } - result.addProperty("androidVersion", androidVersion); - - // Android Version Text - ['ds:5', 1, 2, 140, 1, 1, 0, 0, 1] with helper - Object androidVersionTextObj = getPath(ds5, 1, 2, 140, 1, 1, 0, 0, 1); - String androidVersionText = "Varies with device"; - if (androidVersionTextObj instanceof JsonElement - && ((JsonElement) androidVersionTextObj).isJsonPrimitive()) { - androidVersionText = ((JsonElement) androidVersionTextObj).getAsString(); - } - result.addProperty("androidVersionText", androidVersionText); - - // Developer - ['ds:5', 1, 2, 68, 0] - Object developerObj = getPath(ds5, 1, 2, 68, 0); - if (developerObj instanceof JsonElement && ((JsonElement) developerObj).isJsonPrimitive()) { - result.addProperty("developer", ((JsonElement) developerObj).getAsString()); - } - - // Developer Id - ['ds:5', 1, 2, 68, 1, 4, 2] with function - Object developerIdObj = getPath(ds5, 1, 2, 68, 1, 4, 2); - if (developerIdObj instanceof JsonElement - && ((JsonElement) developerIdObj).isJsonPrimitive()) { - String devUrl = ((JsonElement) developerIdObj).getAsString(); - String devId = extractDeveloperId(devUrl); - if (devId != null) { - result.addProperty("developerId", devId); - } - } - - // Developer Email - ['ds:5', 1, 2, 69, 1, 0] - Object developerEmailObj = getPath(ds5, 1, 2, 69, 1, 0); - if (developerEmailObj instanceof JsonElement - && ((JsonElement) developerEmailObj).isJsonPrimitive()) { - result.addProperty("developerEmail", ((JsonElement) developerEmailObj).getAsString()); - } - - // Developer Website - ['ds:5', 1, 2, 69, 0, 5, 2] - Object developerWebsiteObj = getPath(ds5, 1, 2, 69, 0, 5, 2); - if (developerWebsiteObj instanceof JsonElement - && ((JsonElement) developerWebsiteObj).isJsonPrimitive()) { - result.addProperty("developerWebsite", ((JsonElement) developerWebsiteObj).getAsString()); - } - - // Developer Address - ['ds:5', 1, 2, 69, 2, 0] - Object developerAddressObj = getPath(ds5, 1, 2, 69, 2, 0); - if (developerAddressObj instanceof JsonElement - && ((JsonElement) developerAddressObj).isJsonPrimitive()) { - result.addProperty("developerAddress", ((JsonElement) developerAddressObj).getAsString()); - } - - // Privacy Policy - ['ds:5', 1, 2, 99, 0, 5, 2] - Object privacyPolicyObj = getPath(ds5, 1, 2, 99, 0, 5, 2); - if (privacyPolicyObj instanceof JsonElement - && ((JsonElement) privacyPolicyObj).isJsonPrimitive()) { - result.addProperty("privacyPolicy", ((JsonElement) privacyPolicyObj).getAsString()); - } - - // Genre - ['ds:5', 1, 2, 79, 0, 0, 0] - Object genreObj = getPath(ds5, 1, 2, 79, 0, 0, 0); - if (genreObj instanceof JsonElement && ((JsonElement) genreObj).isJsonPrimitive()) { - result.addProperty("genre", ((JsonElement) genreObj).getAsString()); - } - - // Genre Id - ['ds:5', 1, 2, 79, 0, 0, 2] - Object genreIdObj = getPath(ds5, 1, 2, 79, 0, 0, 2); - if (genreIdObj instanceof JsonElement && ((JsonElement) genreIdObj).isJsonPrimitive()) { - result.addProperty("genreId", ((JsonElement) genreIdObj).getAsString()); - } - - // Icon - ['ds:5', 1, 2, 95, 0, 3, 2] - Object iconObj = getPath(ds5, 1, 2, 95, 0, 3, 2); - if (iconObj instanceof JsonElement && ((JsonElement) iconObj).isJsonPrimitive()) { - result.addProperty("icon", ((JsonElement) iconObj).getAsString()); - } - - // Header Image - ['ds:5', 1, 2, 96, 0, 3, 2] - Object headerImageObj = getPath(ds5, 1, 2, 96, 0, 3, 2); - if (headerImageObj instanceof JsonElement - && ((JsonElement) headerImageObj).isJsonPrimitive()) { - result.addProperty("headerImage", ((JsonElement) headerImageObj).getAsString()); - } - - // Screenshots - ['ds:5', 1, 2, 78, 0] with function - Object screenshotsObj = getPath(ds5, 1, 2, 78, 0); - JsonArray screenshots = extractScreenshots(screenshotsObj); - result.add("screenshots", screenshots); - - // Content Rating - ['ds:5', 1, 2, 9, 0] - Object contentRatingObj = getPath(ds5, 1, 2, 9, 0); - if (contentRatingObj instanceof JsonElement - && ((JsonElement) contentRatingObj).isJsonPrimitive()) { - result.addProperty("contentRating", ((JsonElement) contentRatingObj).getAsString()); - } - - // Ad Supported - ['ds:5', 1, 2, 48] with Boolean function - Object adSupportedObj = getPath(ds5, 1, 2, 48); - boolean adSupported = false; - if (adSupportedObj instanceof JsonElement - && ((JsonElement) adSupportedObj).isJsonPrimitive()) { - adSupported = ((JsonElement) adSupportedObj).getAsInt() == 1; - } - result.addProperty("adSupported", adSupported); - - // Released - ['ds:5', 1, 2, 10, 0] - Object releasedObj = getPath(ds5, 1, 2, 10, 0); - if (releasedObj instanceof JsonElement && ((JsonElement) releasedObj).isJsonPrimitive()) { - result.addProperty("released", ((JsonElement) releasedObj).getAsString()); - } - - // Updated - ['ds:5', 1, 2, 145, 0, 1, 0] with function - Object updatedObj = getPath(ds5, 1, 2, 145, 0, 1, 0); - if (updatedObj instanceof JsonElement && ((JsonElement) updatedObj).isJsonPrimitive()) { - long timestamp = ((JsonElement) updatedObj).getAsLong(); - result.addProperty("updated", timestamp * 1000); - } - - // Version - ['ds:5', 1, 2, 140, 0, 0, 0] with function - Object versionObj = getPath(ds5, 1, 2, 140, 0, 0, 0); - String version = "VARY"; - if (versionObj instanceof JsonElement && ((JsonElement) versionObj).isJsonPrimitive()) { - version = ((JsonElement) versionObj).getAsString(); - if (version == null || version.isEmpty()) { - version = "VARY"; - } - } - result.addProperty("version", version); - - // Recent Changes - ['ds:5', 1, 2, 144, 1, 1] - Object recentChangesObj = getPath(ds5, 1, 2, 144, 1, 1); - if (recentChangesObj instanceof JsonElement - && ((JsonElement) recentChangesObj).isJsonPrimitive()) { - result.addProperty("recentChanges", ((JsonElement) recentChangesObj).getAsString()); - } - - } catch (Exception e) { - // In case of any error, we continue with what we have - e.printStackTrace(); - } - } - - /** - * Extract description text from search array - * @param searchArray The JSON element containing description data - * @return The processed description text - */ - private static String extractDescriptionText(JsonElement searchArray) { - String descriptionHtml = extractDescriptionHtmlLocalized(searchArray); - if (descriptionHtml == null) return ""; - - // This is a simplified version of the descriptionText function - // The original uses cheerio to convert HTML to text while preserving line breaks - // We'll do a basic HTML tag removal with line break preservation - return descriptionHtml.replaceAll("
", "\n").replaceAll("<[^>]+>", "").trim(); - } - - /** - * Extract description HTML localized from search array - * @param searchArray The JSON element containing description data - * @return The HTML localized description - */ - private static String extractDescriptionHtmlLocalized(JsonElement searchArray) { - // This mimics descriptionHtmlLocalized from mappingHelpers.js - // It looks for description translation or original description - JsonElement descriptionTranslation = getPath(searchArray, 12, 0, 0, 1); - JsonElement descriptionOriginal = getPath(searchArray, 72, 0, 1); - - if (descriptionTranslation != null && descriptionTranslation.isJsonPrimitive()) { - return descriptionTranslation.getAsString(); - } else if (descriptionOriginal != null && descriptionOriginal.isJsonPrimitive()) { - return descriptionOriginal.getAsString(); - } else { - return ""; - } - } - - /** - * Build histogram from container - * @param container The histogram data container - * @return JsonObject with histogram data - */ - private static JsonObject buildHistogram(Object container) { - JsonObject histogram = new JsonObject(); - if (container == null) { - histogram.addProperty("1", 0); - histogram.addProperty("2", 0); - histogram.addProperty("3", 0); - histogram.addProperty("4", 0); - histogram.addProperty("5", 0); - return histogram; - } - - JsonElement containerElement = (JsonElement) container; - if (containerElement.isJsonArray()) { - JsonArray arr = containerElement.getAsJsonArray(); - if (arr.size() > 5) { - JsonObject hist = new JsonObject(); - JsonElement one = getPath(arr, 1, 1); - JsonElement two = getPath(arr, 2, 1); - JsonElement three = getPath(arr, 3, 1); - JsonElement four = getPath(arr, 4, 1); - JsonElement five = getPath(arr, 5, 1); - - hist.addProperty("1", one != null && one.isJsonPrimitive() ? one.getAsInt() : 0); - hist.addProperty("2", two != null && two.isJsonPrimitive() ? two.getAsInt() : 0); - hist.addProperty("3", three != null && three.isJsonPrimitive() ? three.getAsInt() : 0); - hist.addProperty("4", four != null && four.isJsonPrimitive() ? four.getAsInt() : 0); - hist.addProperty("5", five != null && five.isJsonPrimitive() ? five.getAsInt() : 0); - return hist; - } - } - - // Default values if histogram data is not available - histogram.addProperty("1", 0); - histogram.addProperty("2", 0); - histogram.addProperty("3", 0); - histogram.addProperty("4", 0); - histogram.addProperty("5", 0); - return histogram; - } - - /** - * Normalize Android version text - * @param androidVersionText The original Android version text - * @return Normalized Android version - */ - private static String normalizeAndroidVersion(String androidVersionText) { - if (androidVersionText == null) return "VARY"; - - String number = androidVersionText.split(" ")[0]; - try { - double num = Double.parseDouble(number); - return Double.toString(num); - } catch (NumberFormatException e) { - return "VARY"; - } - } - - /** - * Extract developer ID from developer URL - * @param devUrl The developer URL - * @return The extracted developer ID - */ - private static String extractDeveloperId(String devUrl) { - if (devUrl != null && devUrl.contains("id=")) { - String[] parts = devUrl.split("id="); - if (parts.length > 1) { - return parts[1]; - } - } - return null; - } - - /** - * Extract screenshots from screenshots data - * @param screenshotsObj The screenshots data object - * @return JsonArray of screenshot URLs - */ - private static JsonArray extractScreenshots(Object screenshotsObj) { - JsonArray screenshots = new JsonArray(); - if (screenshotsObj == null) return screenshots; - - JsonElement screenshotsElement = (JsonElement) screenshotsObj; - if (screenshotsElement.isJsonArray()) { - JsonArray array = screenshotsElement.getAsJsonArray(); - for (int i = 0; i < array.size(); i++) { - JsonElement screenshotItem = array.get(i); - JsonElement url = getPath(screenshotItem, 3, 2); - if (url != null && url.isJsonPrimitive()) { - screenshots.add(url.getAsString()); - } - } - } - return screenshots; - } - - /** - * Get value at specified path in nested JSON structure - * @param element The root JSON element - * @param path The path indices - * @return The value at the specified path, or null if not found - */ - private static JsonElement getPath(JsonElement element, int... path) { - if (element == null) return null; - - JsonElement current = element; - - for (int index : path) { - if (current == null) return null; - - if (current.isJsonArray()) { - JsonArray array = current.getAsJsonArray(); - if (index < array.size()) { - current = array.get(index); - } else { - return null; - } - } else if (current.isJsonObject()) { - // For objects, we might need a different approach - return null; - } else { - return null; - } - } - - return current; - } - - /** - * Main method for testing - * @param args Command line arguments - */ - public static void main(String[] args) { - try { - // if (args.length < 1) { - // System.out.println("Usage: java GooglePlayAppScraper "); - // System.out.println("Example: java GooglePlayAppScraper com.android.chrome"); - // return; - // } - - // String appId = args[0]; - String appId = "vn.kvtm.js"; - System.out.println("Fetching details for app: " + appId); - JsonObject appDetails = getAppDetails(appId); - - // Pretty print the JSON response - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - System.out.println(gson.toJson(appDetails)); - - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - e.printStackTrace(); - } - } -} diff --git a/src/main/java/com/miti99/storescraperbot/util/RequestUtil.java b/src/main/java/com/miti99/storescraperbot/util/RequestUtil.java deleted file mode 100644 index 7715348..0000000 --- a/src/main/java/com/miti99/storescraperbot/util/RequestUtil.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.miti99.storescraperbot.util; - -import com.google.gson.reflect.TypeToken; -import java.lang.reflect.Type; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class RequestUtil { - private static final Type MAP_STRING_STRING_TYPE = - new TypeToken>() {}.getType(); - - private static Map objToMapStringString(Object obj) { - return GsonUtil.GSON.fromJson(GsonUtil.GSON.toJson(obj), MAP_STRING_STRING_TYPE); - } - - public static String makeGetParams(Map params) { - return params.entrySet().stream() - .filter(entry -> entry.getValue() != null) - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .collect(Collectors.joining("&")); - } - - public static String makeGetParams(Object obj) { - return makeGetParams(objToMapStringString(obj)); - } -} diff --git a/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java b/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java index 3b8c68a..11ec689 100644 --- a/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java +++ b/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java @@ -1,27 +1,28 @@ package com.miti99.storescraperbot.api.apple; -import static org.junit.jupiter.api.Assertions.*; - +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; import com.miti99.storescraperbot.util.GsonUtil; import org.junit.jupiter.api.Test; class AppStoreScraperTest { @Test void testComMptKvtm() { - var response = AppStoreScraper.app("com.mpt.kvtm"); + var request = new AppleAppRequest("com.mpt.kvtm", "vn"); + var response = AppStoreScraper.app(request); System.out.println(GsonUtil.toJson(response)); } @Test void testComMptBuraco() { - // var response = AppStoreScraper.app("com.mpt.buraco"); - var response = AppStoreScraper.app(1638264682); + var request = new AppleAppRequest("com.mpt.buraco", "mx"); + var response = AppStoreScraper.app(request); System.out.println(GsonUtil.toJson(response)); } @Test void testComMptDoudizhu() { - var response = AppStoreScraper.app("com.mpt.doudizhu"); + var request = new AppleAppRequest("com.mpt.doudizhu", "hk"); + var response = AppStoreScraper.app(request); System.out.println(GsonUtil.toJson(response)); } } diff --git a/src/test/java/com/miti99/storescraperbot/api/old/google/GooglePlayScraperTest.java b/src/test/java/com/miti99/storescraperbot/api/google/GooglePlayScraperTest.java similarity index 79% rename from src/test/java/com/miti99/storescraperbot/api/old/google/GooglePlayScraperTest.java rename to src/test/java/com/miti99/storescraperbot/api/google/GooglePlayScraperTest.java index 2915290..487d32e 100644 --- a/src/test/java/com/miti99/storescraperbot/api/old/google/GooglePlayScraperTest.java +++ b/src/test/java/com/miti99/storescraperbot/api/google/GooglePlayScraperTest.java @@ -1,7 +1,6 @@ -package com.miti99.storescraperbot.api.old.google; +package com.miti99.storescraperbot.api.google; -import com.miti99.storescraperbot.api.old.google.GooglePlayScraper; -import com.miti99.storescraperbot.api.old.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; import com.miti99.storescraperbot.util.GsonUtil; import org.junit.jupiter.api.Test; diff --git a/src/test/java/com/miti99/storescraperbot/api/old/apple/AppStoreScraperTest.java b/src/test/java/com/miti99/storescraperbot/api/old/apple/AppStoreScraperTest.java deleted file mode 100644 index 2b2cce2..0000000 --- a/src/test/java/com/miti99/storescraperbot/api/old/apple/AppStoreScraperTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.miti99.storescraperbot.api.old.apple; - -import com.miti99.storescraperbot.api.old.apple.AppStoreScraper; -import com.miti99.storescraperbot.api.old.apple.request.AppleAppRequest; -import com.miti99.storescraperbot.util.GsonUtil; -import org.junit.jupiter.api.Test; - -class AppStoreScraperTest { - @Test - void testComMptKvtm() { - var request = new AppleAppRequest("com.mpt.kvtm", "vn"); - var response = AppStoreScraper.app(request); - System.out.println(GsonUtil.toJson(response)); - } - - @Test - void testComMptBuraco() { - var request = new AppleAppRequest("com.mpt.buraco", "mx"); - var response = AppStoreScraper.app(request); - System.out.println(GsonUtil.toJson(response)); - } - - @Test - void testComMptDoudizhu() { - var request = new AppleAppRequest("com.mpt.doudizhu", "hk"); - var response = AppStoreScraper.app(request); - System.out.println(GsonUtil.toJson(response)); - } -}