Skip to content

Commit

Permalink
Merge branch 'aniyomiorg:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@Dark25
Dark25 authored Mar 26, 2024
2 parents 3ede337 + 0b59228 commit 72b2382
Show file tree
Hide file tree
Showing 32 changed files with 981 additions and 758 deletions.
122 changes: 14 additions & 108 deletions ...ractor/src/main/java/eu/kanade/tachiyomi/lib/googledriveextractor/GoogleDriveExtractor.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,11 @@ package eu.kanade.tachiyomi.lib.googledriveextractor

import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.internal.commonEmptyRequestBody

class GoogleDriveExtractor(private val client: OkHttpClient, private val headers: Headers) {

Expand All @@ -20,28 +16,21 @@ class GoogleDriveExtractor(private val client: OkHttpClient, private val headers

private val cookieList = client.cookieJar.loadForRequest("https://drive.google.com".toHttpUrl())

private val noRedirectClient = OkHttpClient.Builder()
.followRedirects(false)
.build()

fun videosFromUrl(itemUrl: String, videoName: String = "Video"): List<Video> {
val cookieJar = GDriveCookieJar()

cookieJar.saveFromResponse("https://drive.google.com".toHttpUrl(), cookieList)

fun videosFromUrl(itemId: String, videoName: String = "Video"): List<Video> {
val url = "https://drive.usercontent.google.com/download?id=$itemId"
val docHeaders = headers.newBuilder().apply {
add("Accept", ACCEPT)
add("Connection", "keep-alive")
add("Cookie", cookieList.toStr())
add("Host", "drive.google.com")
}.build()

val docResp = noRedirectClient.newCall(
GET(itemUrl, headers = docHeaders),
val docResp = client.newCall(
GET(url, docHeaders)
).execute()

if (docResp.isRedirect) {
return videoFromRedirect(itemUrl, videoName, "", cookieJar)
if (!docResp.peekBody(15).string().equals("<!DOCTYPE html>", true)) {
return listOf(
Video(url, videoName, url, docHeaders)
)
}

val document = docResp.asJsoup()
Expand All @@ -50,101 +39,18 @@ class GoogleDriveExtractor(private val client: OkHttpClient, private val headers
?.let { " ${it.ownText().trim()} " }
?: ""

val downloadUrl = document.selectFirst("form#download-form")?.attr("action") ?: return emptyList()
val postHeaders = headers.newBuilder().apply {
add("Accept", ACCEPT)
add("Content-Type", "application/x-www-form-urlencoded")
set("Cookie", client.cookieJar.loadForRequest("https://drive.google.com".toHttpUrl()).toStr())
add("Host", "drive.google.com")
add("Referer", "https://drive.google.com/")
}.build()

val newUrl = noRedirectClient.newCall(
POST(downloadUrl, headers = postHeaders, body = commonEmptyRequestBody),
).execute().use { it.headers["location"] ?: downloadUrl }

return videoFromRedirect(newUrl, videoName, itemSize, cookieJar)
}

private fun videoFromRedirect(
downloadUrl: String,
videoName: String,
itemSize: String,
cookieJar: GDriveCookieJar,
): List<Video> {
var newUrl = downloadUrl

val newHeaders = headers.newBuilder().apply {
add("Accept", ACCEPT)
set("Cookie", cookieJar.loadForRequest(newUrl.toHttpUrl()).toStr())
set("Host", newUrl.toHttpUrl().host)
add("Referer", "https://drive.google.com/")
}.build()

var newResp = noRedirectClient.newCall(
GET(newUrl, headers = newHeaders),
).execute()

var redirectCounter = 1
while (newResp.isRedirect && redirectCounter < 15) {
val setCookies = newResp.headers("Set-Cookie").mapNotNull { Cookie.parse(newResp.request.url, it) }
cookieJar.saveFromResponse(newResp.request.url, setCookies)

newUrl = newResp.headers["location"]!!
newResp.close()

val newHeaders = headers.newBuilder().apply {
add("Accept", ACCEPT)
set("Cookie", cookieJar.loadForRequest(newUrl.toHttpUrl()).toStr())
set("Host", newUrl.toHttpUrl().host)
add("Referer", "https://drive.google.com/")
}.build()

newResp = noRedirectClient.newCall(
GET(newUrl, headers = newHeaders),
).execute()
redirectCounter += 1
}

val videoUrl = newResp.use { it.request.url }

val videoHeaders = headers.newBuilder().apply {
add("Accept", ACCEPT)
set("Cookie", cookieJar.loadForRequest(videoUrl).toStr())
set("Host", videoUrl.host)
add("Referer", "https://drive.google.com/")
}.build()
val videoUrl = url.toHttpUrl().newBuilder().apply {
document.select("input[type=hidden]").forEach {
setQueryParameter(it.attr("name"), it.attr("value"))
}
}.build().toString()

return listOf(
Video(
videoUrl.toString(),
videoName + itemSize,
videoUrl.toString(),
headers = videoHeaders,
),
Video(videoUrl, videoName + itemSize, videoUrl, docHeaders)
)
}

private fun List<Cookie>.toStr(): String {
return this.joinToString("; ") { "${it.name}=${it.value}" }
}
}

class GDriveCookieJar : CookieJar {

private val cookieStore = mutableMapOf<String, MutableList<Cookie>>()

// Append rather than overwrite, what could go wrong?
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
val oldCookies = (cookieStore[url.host] ?: emptyList()).filter { c ->
!cookies.any { t -> c.name == t.name }
}
cookieStore[url.host] = (oldCookies + cookies).toMutableList()
}

override fun loadForRequest(url: HttpUrl): List<Cookie> {
val cookies = cookieStore[url.host] ?: emptyList()

return cookies.filter { it.expiresAt >= System.currentTimeMillis() }
}
}
86 changes: 45 additions & 41 deletions lib/javcoverfetcher/src/main/java/eu/kanade/tachiyomi/lib/javcoverfetcher/JavCoverFetcher.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,15 @@ import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.FormBody
import okhttp3.Interceptor
import okhttp3.Response
import okhttp3.Request
import okhttp3.internal.commonEmptyHeaders
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.io.IOException

object JavCoverFetcher {

private val CLIENT by lazy {
Injekt.get<NetworkHelper>().client.newBuilder()
.addInterceptor(::amazonAgeVerifyIntercept)
.build()
}

private val HEADERS by lazy {
commonEmptyHeaders.newBuilder()
.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36")
.build()
}

private fun amazonAgeVerifyIntercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)

if (!request.url.host.contains("amazon.co.jp") || !response.request.url.pathSegments.contains("black-curtain")) {
return response
}

val document = response.asJsoup()
val targetUrl = document.selectFirst("#black-curtain-yes-button a")?.attr("abs:href")
?: throw IOException("Failed to bypass Amazon Age Gate")

val newRequest = request.newBuilder().apply {
url(targetUrl)
}.build()

return chain.proceed(newRequest)
Injekt.get<NetworkHelper>().client
}

/**
Expand Down Expand Up @@ -89,7 +60,7 @@ object JavCoverFetcher {
private fun getJPTitleFromID(javId: String): String? {
val url = "https://www.javlibrary.com/ja/vl_searchbyid.php?keyword=$javId"

val request = GET(url, HEADERS)
val request = GET(url, commonEmptyHeaders)

val response = CLIENT.newCall(request).execute()

Expand All @@ -100,7 +71,7 @@ object JavCoverFetcher {
val targetUrl = document.selectFirst(".videos a[href*=\"?v=\"]")?.attr("abs:href")
?: return null

document = CLIENT.newCall(GET(targetUrl, HEADERS)).execute().asJsoup()
document = CLIENT.newCall(GET(targetUrl, commonEmptyHeaders)).execute().asJsoup()
}

val dirtyTitle = document.selectFirst(".post-title")?.text()
Expand All @@ -111,13 +82,27 @@ object JavCoverFetcher {
}

private fun getDDGSearchResult(jpTitle: String): String? {
val url = "https://lite.duckduckgo.com/lite"
val url = "https://lite.duckduckgo.com/lite/"

val form = FormBody.Builder()
.add("q", "site:amazon.co.jp inurl:/dp/$jpTitle")
.build()

val request = POST(url, HEADERS, form)
val headers = commonEmptyHeaders.newBuilder().apply {
add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
add("Host", "lite.duckduckgo.com")
add("Referer", "https://lite.duckduckgo.com/")
add("Origin", "https://lite.duckduckgo.com")
add("Accept-Language", "en-US,en;q=0.5")
add("DNT", "1")
add("Sec-Fetch-Dest", "document")
add("Sec-Fetch-Mode", "navigate")
add("Sec-Fetch-Site", "same-origin")
add("Sec-Fetch-User", "?1")
add("TE", "trailers")
}.build()

val request = POST(url, headers, form)

val response = CLIENT.newCall(request).execute()

Expand All @@ -127,22 +112,41 @@ object JavCoverFetcher {
}

private fun getHDCoverFromAmazonUrl(amazonUrl: String): String? {
val request = GET(amazonUrl, HEADERS)
val basicCoverUrl = "https://m.media-amazon.com/images/P/%s.01.MAIN._SCRM_.jpg"
val asinRegex = Regex("""/dp/(\w+)""")

val response = CLIENT.newCall(request).execute()
val asin = asinRegex.find(amazonUrl)?.groupValues?.get(1)
?: return null

val document = response.asJsoup()
var cover = basicCoverUrl.replace("%s", asin)

if (!checkCover(cover)) {
cover = cover.replace(".01.", ".")
}

return cover
}

private fun checkCover(cover: String): Boolean {
return getContentLength(cover) > 100
}

private fun getContentLength(url: String): Long {
val request = Request.Builder()
.head()
.url(url)
.build()

val smallImage = document.selectFirst("#landingImage")?.attr("src")
val res = CLIENT.newCall(request).execute()

return smallImage?.replace(Regex("""(\._\w+_\.jpg)"""), ".jpg")
return res.use { it.headers["content-length"] }?.toLongOrNull() ?: 0
}

fun addPreferenceToScreen(screen: PreferenceScreen) {
SwitchPreferenceCompat(screen.context).apply {
key = "JavCoverFetcherPref"
title = "Fetch HD covers from Amazon"
summary = "Attempts to fetch HD covers from Amazon.\nMay result in incorrect cover."
summary = "Attempts to fetch vertical HD covers from Amazon.\nMay result in incorrect cover."
setDefaultValue(false)
}.also(screen::addPreference)
}
Expand Down
2 changes: 1 addition & 1 deletion src/all/googledrive/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ext {
extName = 'Google Drive'
extClass = '.GoogleDrive'
extVersionCode = 14
extVersionCode = 15
}

apply from: "$rootDir/common.gradle"
Expand Down
2 changes: 1 addition & 1 deletion src/all/googledrive/src/eu/kanade/tachiyomi/animeextension/all/googledrive/GoogleDrive.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ class GoogleDrive : ConfigurableAnimeSource, AnimeHttpSource() {
// ============================ Video Links =============================

override suspend fun getVideoList(episode: SEpisode): List<Video> =
GoogleDriveExtractor(client, headers).videosFromUrl(episode.url)
GoogleDriveExtractor(client, headers).videosFromUrl(episode.url.substringAfter("?id="))

// ============================= Utilities ==============================

Expand Down
2 changes: 1 addition & 1 deletion src/all/javguru/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ext {
extName = 'Jav Guru'
extClass = '.JavGuru'
extVersionCode = 13
extVersionCode = 14
isNsfw = true
}

Expand Down
19 changes: 10 additions & 9 deletions src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/JavGuru.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,13 @@ class JavGuru : AnimeHttpSource(), ConfigurableAnimeSource {
return redirectUrl
}

private val streamWishExtractor by lazy { StreamWishExtractor(client, headers) }
private val streamWishExtractor by lazy {
val swHeaders = headersBuilder()
.set("Referer", "$baseUrl/")
.build()

StreamWishExtractor(client, swHeaders)
}
private val streamTapeExtractor by lazy { StreamTapeExtractor(client) }
private val doodExtractor by lazy { DoodExtractor(client) }
private val mixDropExtractor by lazy { MixDropExtractor(client) }
Expand All @@ -281,19 +287,19 @@ class JavGuru : AnimeHttpSource(), ConfigurableAnimeSource {

private fun getVideos(hosterUrl: String): List<Video> {
return when {
hosterUrl.contains("javplaya") -> {
listOf("javplaya", "javclan").any { it in hosterUrl } -> {
streamWishExtractor.videosFromUrl(hosterUrl)
}

hosterUrl.contains("streamtape") -> {
streamTapeExtractor.videoFromUrl(hosterUrl).let(::listOfNotNull)
}

hosterUrl.contains("dood") -> {
listOf("dood", "ds2play").any { it in hosterUrl } -> {
doodExtractor.videosFromUrl(hosterUrl)
}

MIXDROP_DOMAINS.any { it in hosterUrl } -> {
listOf("mixdrop", "mixdroop").any { it in hosterUrl } -> {
mixDropExtractor.videoFromUrl(hosterUrl)
}

Expand Down Expand Up @@ -364,11 +370,6 @@ class JavGuru : AnimeHttpSource(), ConfigurableAnimeSource {
private val IFRAME_OLID_REGEX = Regex("""var OLID = '([^']+)'""")
private val IFRAME_OLID_URL = Regex("""src="([^"]+)"""")

private val MIXDROP_DOMAINS = listOf(
"mixdrop",
"mixdroop",
)

private const val PREF_QUALITY = "preferred_quality"
private const val PREF_QUALITY_TITLE = "Preferred quality"
private const val PREF_QUALITY_DEFAULT = "720"
Expand Down
2 changes: 1 addition & 1 deletion src/all/missav/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ext {
extName = 'MissAV'
extClass = '.MissAV'
extVersionCode = 9
extVersionCode = 10
isNsfw = true
}

Expand Down
2 changes: 1 addition & 1 deletion src/all/missav/src/eu/kanade/tachiyomi/animeextension/all/missav/MissAV.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ class MissAV : AnimeHttpSource(), ConfigurableAnimeSource {

val masterPlaylist = playlists.substringAfter("source=\"").substringBefore("\";")

return playlistExtractor.extractFromHls(masterPlaylist)
return playlistExtractor.extractFromHls(masterPlaylist, referer = "$baseUrl/")
}

override fun List<Video>.sort(): List<Video> {
Expand Down
Loading

0 comments on commit 72b2382

Please sign in to comment.