Initial commit.
This commit is contained in:
Hellings
commit
89a7c8eaef
90 changed files with 3920 additions and 0 deletions
12
.gitattributes
vendored
Normal file
12
.gitattributes
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
#
|
||||||
|
# https://help.github.com/articles/dealing-with-line-endings/
|
||||||
|
#
|
||||||
|
# Linux start script should use lf
|
||||||
|
/gradlew text eol=lf
|
||||||
|
|
||||||
|
# These are Windows script files and should use crlf
|
||||||
|
*.bat text eol=crlf
|
||||||
|
|
||||||
|
# Binary files should be left untouched
|
||||||
|
*.jar binary
|
||||||
|
|
||||||
12
.gitignore
vendored
Normal file
12
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
# Ignore Gradle project-specific cache directory
|
||||||
|
.gradle
|
||||||
|
|
||||||
|
# Ignore Gradle build output directory
|
||||||
|
build
|
||||||
|
|
||||||
|
# Ignore Kotlin plugin data
|
||||||
|
.kotlin
|
||||||
|
|
||||||
|
# Node credentials and deployment-specific endpoints
|
||||||
|
node.yaml
|
||||||
|
deploy/*.yaml
|
||||||
16
Justfile
Normal file
16
Justfile
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
|
||||||
|
|
||||||
|
build:
|
||||||
|
./gradlew build
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
./gradlew spotlessApply
|
||||||
|
|
||||||
|
check:
|
||||||
|
./gradlew spotlessCheck build
|
||||||
|
|
||||||
|
native:
|
||||||
|
./gradlew nativeImage
|
||||||
|
|
||||||
|
deploy host user:
|
||||||
|
scripts/deploy-ssh.sh {{host}} {{user}}
|
||||||
12
README.md
Normal file
12
README.md
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
# CommonIP
|
||||||
|
|
||||||
|
CommonIP, not to be confused with [CommonApp](https://commonapp.org), is a decentralized geolocation network that specifically avoids proprietary geolocation services whenever possible. It's written in the ever-so-portable Kotlin. Use it from here, use it from there, use it from anywhere!
|
||||||
|
|
||||||
|
# donate
|
||||||
|
|
||||||
|
Please donate to me to keep this project and its (centralized) [geofeeds](https://ai.crit.rip/automatedintelligence/geofeeds) alive.
|
||||||
|
|
||||||
|
- `eth` 0xF30C281Aa2CD94618D902AE5bb9101E9F5385fE3
|
||||||
|
- `sol` 99zJPASUz9PKLFaJWrxHCEtLZxeEePYpgZ74Ly9YNUHy
|
||||||
|
- `paypal` +1 (267) 806-2744 - Stephen Hellings
|
||||||
|
- `zelle` +1 (267) 806-2744 - STEPHEN HELLINGS
|
||||||
32
app/build.gradle.kts
Normal file
32
app/build.gradle.kts
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
import buildlogic.GraalNativeImageTask
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-application-conventions")
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation("org.apache.commons:commons-text")
|
||||||
|
implementation(project(":packages:serialize"))
|
||||||
|
implementation(project(":packages:rawr"))
|
||||||
|
implementation(project(":proprietary"))
|
||||||
|
implementation(project(":packages:ml"))
|
||||||
|
implementation(project(":packages:decentraland"))
|
||||||
|
}
|
||||||
|
|
||||||
|
application {
|
||||||
|
mainClass = "rip.crit.commonip.app.MainKtKt"
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named<JavaExec>("run") {
|
||||||
|
standardInput = System.`in`
|
||||||
|
}
|
||||||
|
|
||||||
|
val nativeImage =
|
||||||
|
tasks.register<GraalNativeImageTask>("nativeImage") {
|
||||||
|
group = "distribution"
|
||||||
|
description = "Builds a GraalVM native executable with the active Gradle toolchain."
|
||||||
|
dependsOn(tasks.named("classes"))
|
||||||
|
classpath.set(sourceSets.main.get().runtimeClasspath.files.map { it.absolutePath })
|
||||||
|
mainClass.set("rip.crit.commonip.app.MainKtKt")
|
||||||
|
output.set(layout.buildDirectory.file("native/commonip"))
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
package rip.crit.commonip.app
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import rip.crit.commonip.decentraland.git.RepositoryTrust
|
||||||
|
import rip.crit.commonip.decentraland.kademlia.KademliaNode
|
||||||
|
import rip.crit.commonip.decentraland.kademlia.NodeRegistry
|
||||||
|
import rip.crit.commonip.ml.GeoLocation as MlGeoLocation
|
||||||
|
import rip.crit.commonip.ml.LocationEstimator
|
||||||
|
import rip.crit.commonip.proprietary.OpenGeoFeed
|
||||||
|
import rip.crit.commonip.proprietary.StarlinkGeoFeed
|
||||||
|
import rip.crit.commonip.rawr.providers.RirGeoFeedLoader
|
||||||
|
import rip.crit.commonip.rawr.providers.RirMetadataField
|
||||||
|
import rip.crit.commonip.rawr.types.Cidr
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
class ClusterGeolocation(private val registry: NodeRegistry, bind: InetSocketAddress) :
|
||||||
|
AutoCloseable {
|
||||||
|
private val feeds = ConcurrentHashMap<String, RepositoryTrust>()
|
||||||
|
private val local = ConcurrentHashMap<String, GeoLocation>()
|
||||||
|
private val dht = KademliaNode(bind, registry)
|
||||||
|
|
||||||
|
fun start(seeds: List<InetSocketAddress> = emptyList()) {
|
||||||
|
dht.start()
|
||||||
|
seeds.forEach(dht::bootstrap)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun addFeed(url: String, trust: RepositoryTrust = RepositoryTrust.COMMUNITY) {
|
||||||
|
feeds[url] = trust
|
||||||
|
}
|
||||||
|
|
||||||
|
fun addStarlinkFeed() {
|
||||||
|
feeds[StarlinkGeoFeed.URL] = RepositoryTrust.TAINTED
|
||||||
|
}
|
||||||
|
|
||||||
|
fun pullRirRegistryObject(
|
||||||
|
source: String,
|
||||||
|
registryObject: String,
|
||||||
|
fields: Set<RirMetadataField>,
|
||||||
|
trust: RepositoryTrust = RepositoryTrust.COMMUNITY,
|
||||||
|
) {
|
||||||
|
RirGeoFeedLoader(source, fields).pull(registryObject).forEach { feed ->
|
||||||
|
feed.records.forEach { record ->
|
||||||
|
local[record.prefix] = record
|
||||||
|
dht.store(
|
||||||
|
"${if (trust == RepositoryTrust.TAINTED) "commonip/tainted-full" else "commonip/community"}:${record.prefix}",
|
||||||
|
record.prefix,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
registry.publish(
|
||||||
|
"local",
|
||||||
|
"rir geofeed pulled",
|
||||||
|
"$source yielded ${feed.records.size} records",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun pullAll() {
|
||||||
|
feeds.forEach { (url, trust) ->
|
||||||
|
runCatching { OpenGeoFeed().pull(url).records }
|
||||||
|
.onSuccess { records ->
|
||||||
|
records.forEach { record ->
|
||||||
|
local[record.prefix] = record
|
||||||
|
dht.store(
|
||||||
|
"${if (trust == RepositoryTrust.TAINTED) "commonip/tainted-full" else "commonip/community"}:${record.prefix}",
|
||||||
|
record.prefix,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
registry.publish(
|
||||||
|
"local",
|
||||||
|
"${trust.name.lowercase()} feed pulled",
|
||||||
|
"$url yielded ${records.size} records",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.onFailure { registry.publish("local", "feed pull failed", "$url ${it.message}") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun lookup(address: String): GeoLocation? =
|
||||||
|
local.values
|
||||||
|
.filter { Cidr.contains(it.prefix, address) }
|
||||||
|
.maxByOrNull { Cidr.prefixLength(it.prefix) }
|
||||||
|
|
||||||
|
fun estimate(address: String) =
|
||||||
|
LocationEstimator()
|
||||||
|
.estimate(
|
||||||
|
local.values
|
||||||
|
.filter { Cidr.contains(it.prefix, address) }
|
||||||
|
.map { MlGeoLocation(it.latitude, it.longitude, it.confidence) }
|
||||||
|
)
|
||||||
|
|
||||||
|
override fun close() = dht.close()
|
||||||
|
}
|
||||||
116
app/src/main/kotlin/rip/crit/commonip/app/MainKt.kt
Normal file
116
app/src/main/kotlin/rip/crit/commonip/app/MainKt.kt
Normal file
|
|
@ -0,0 +1,116 @@
|
||||||
|
package rip.crit.commonip.app
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.nio.file.Files
|
||||||
|
import rip.crit.commonip.decentraland.git.NodeYamlReader
|
||||||
|
import rip.crit.commonip.decentraland.SshDashboardService
|
||||||
|
import rip.crit.commonip.decentraland.kademlia.NetworkNode
|
||||||
|
import rip.crit.commonip.decentraland.kademlia.NodeRegistry
|
||||||
|
import rip.crit.commonip.decentraland.kademlia.NodeState
|
||||||
|
|
||||||
|
fun main() = CommonIpNode().run()
|
||||||
|
|
||||||
|
private class CommonIpNode {
|
||||||
|
private val registry = NodeRegistry()
|
||||||
|
private val cluster =
|
||||||
|
ClusterGeolocation(
|
||||||
|
registry,
|
||||||
|
InetSocketAddress(System.getenv("COMMONIP_DHT_PORT")?.toIntOrNull() ?: 7400),
|
||||||
|
)
|
||||||
|
|
||||||
|
fun run() {
|
||||||
|
val configPath = Path.of(System.getenv("COMMONIP_NODE_CONFIG") ?: "/etc/commonip/node.yaml")
|
||||||
|
val configuredBootstrap = if (Files.isRegularFile(configPath)) NodeYamlReader.parse(Files.readString(configPath)).bootstrap else emptyList()
|
||||||
|
cluster.start((configuredBootstrap + System.getenv("COMMONIP_BOOTSTRAP")?.split(',').orEmpty()).mapNotNull(::parseEndpoint).distinct())
|
||||||
|
System.getenv("COMMONIP_GEOFEEDS")
|
||||||
|
?.split(',')
|
||||||
|
?.filter(String::isNotBlank)
|
||||||
|
?.forEach(cluster::addFeed)
|
||||||
|
if (System.getenv("COMMONIP_ENABLE_TAINTED_STARLINK") == "true") cluster.addStarlinkFeed()
|
||||||
|
cluster.pullAll()
|
||||||
|
val sshService =
|
||||||
|
System.getenv("COMMONIP_SSH_PASSWD")?.let { password ->
|
||||||
|
SshDashboardService(
|
||||||
|
System.getenv("COMMONIP_SSH_PORT")?.toIntOrNull() ?: 2222,
|
||||||
|
Path.of(System.getenv("COMMONIP_SSH_HOST_KEY") ?: "/opt/commonip/hostkey.ser"),
|
||||||
|
System.getenv("COMMONIP_SSH_USER") ?: "commonip",
|
||||||
|
password.toCharArray(),
|
||||||
|
) {
|
||||||
|
registry.snapshot().joinToString("\n") { "${it.id} ${it.state}" }
|
||||||
|
}
|
||||||
|
.also { it.start() }
|
||||||
|
}
|
||||||
|
if (System.console() == null) {
|
||||||
|
while (true) {
|
||||||
|
Thread.sleep(60_000)
|
||||||
|
cluster.pullAll()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sshService?.close()
|
||||||
|
while (true) {
|
||||||
|
render()
|
||||||
|
print("commonip/network> ")
|
||||||
|
val command = readlnOrNull()?.trim()?.lowercase() ?: return
|
||||||
|
when (command) {
|
||||||
|
"q",
|
||||||
|
"quit",
|
||||||
|
"exit" -> return
|
||||||
|
"r",
|
||||||
|
"refresh" -> cluster.pullAll()
|
||||||
|
"l",
|
||||||
|
"lookup" -> lookup()
|
||||||
|
"e",
|
||||||
|
"estimate" -> estimate()
|
||||||
|
else -> println("Commands are r refresh, l lookup, e estimate, q quit.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseEndpoint(value: String): InetSocketAddress? =
|
||||||
|
value.trim().split(':', limit = 2).let { parts ->
|
||||||
|
parts.getOrNull(1)?.toIntOrNull()?.let { InetSocketAddress(parts[0], it) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun render() {
|
||||||
|
print("\u001b[H\u001b[2J")
|
||||||
|
val nodes = registry.snapshot()
|
||||||
|
println()
|
||||||
|
nodes.chunked(8).forEach { row ->
|
||||||
|
println(row.joinToString(" ") { node -> "${status(node)} ${node.id}" })
|
||||||
|
}
|
||||||
|
println()
|
||||||
|
registry.feed().forEach { event ->
|
||||||
|
println(
|
||||||
|
"${event.at.toString().substring(11, 19)} ${event.nodeId} ${event.action} ${event.detail}"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun status(node: NetworkNode): String =
|
||||||
|
when (node.state) {
|
||||||
|
NodeState.ONLINE -> "\u001b[32m●\u001b[0m"
|
||||||
|
NodeState.DEGRADED -> "\u001b[33m●\u001b[0m"
|
||||||
|
NodeState.OFFLINE -> "\u001b[90m●\u001b[0m"
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun lookup() {
|
||||||
|
print("IPv4 address> ")
|
||||||
|
val address = readlnOrNull()?.trim().orEmpty()
|
||||||
|
val match = cluster.lookup(address)
|
||||||
|
registry.publish("local", "lookup", match?.prefix ?: "no matching CIDR for $address")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun estimate() {
|
||||||
|
print("CIDR or IPv4 address> ")
|
||||||
|
val address = readlnOrNull()?.trim().orEmpty()
|
||||||
|
val result = cluster.estimate(address)
|
||||||
|
registry.publish(
|
||||||
|
"local",
|
||||||
|
"estimate",
|
||||||
|
result?.let {
|
||||||
|
"%.4f, %.4f confidence %.2f".format(it.latitude, it.longitude, it.confidence)
|
||||||
|
} ?: "no coordinate observations",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
18
build-logic/build.gradle.kts
Normal file
18
build-logic/build.gradle.kts
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
`kotlin-dsl`
|
||||||
|
}
|
||||||
|
|
||||||
|
repositories {
|
||||||
|
gradlePluginPortal()
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation(libs.kotlin.gradle.plugin)
|
||||||
|
implementation("com.diffplug.spotless:com.diffplug.spotless.gradle.plugin:8.10.2")
|
||||||
|
}
|
||||||
14
build-logic/settings.gradle.kts
Normal file
14
build-logic/settings.gradle.kts
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This settings file is used to specify which projects to include in your build-logic build.
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
dependencyResolutionManagement {
|
||||||
|
versionCatalogs {
|
||||||
|
create("libs", { from(files("../gradle/libs.versions.toml")) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rootProject.name = "build-logic"
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-common-conventions")
|
||||||
|
|
||||||
|
application
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("org.jetbrains.kotlin.jvm")
|
||||||
|
id("com.diffplug.spotless")
|
||||||
|
}
|
||||||
|
|
||||||
|
repositories {
|
||||||
|
mavenCentral()
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
constraints {
|
||||||
|
implementation("org.apache.commons:commons-text:1.14.0")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
testing {
|
||||||
|
suites {
|
||||||
|
val test = named<JvmTestSuite>("test") {
|
||||||
|
useJUnitJupiter("6.0.1")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named("check") {
|
||||||
|
dependsOn("spotlessCheck")
|
||||||
|
}
|
||||||
|
|
||||||
|
java {
|
||||||
|
toolchain {
|
||||||
|
languageVersion = JavaLanguageVersion.of(25)
|
||||||
|
vendor = JvmVendorSpec.GRAAL_VM
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spotless {
|
||||||
|
kotlin {
|
||||||
|
target("src/**/*.kt")
|
||||||
|
ktfmt()
|
||||||
|
trimTrailingWhitespace()
|
||||||
|
endWithNewline()
|
||||||
|
}
|
||||||
|
kotlinGradle {
|
||||||
|
target("*.gradle.kts")
|
||||||
|
ktfmt()
|
||||||
|
trimTrailingWhitespace()
|
||||||
|
endWithNewline()
|
||||||
|
}
|
||||||
|
}
|
||||||
17
build-logic/src/main/kotlin/buildlogic.kotlin-fmt.gradle.kts
Normal file
17
build-logic/src/main/kotlin/buildlogic.kotlin-fmt.gradle.kts
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
plugins {
|
||||||
|
id("com.diffplug.spotless")
|
||||||
|
}
|
||||||
|
|
||||||
|
spotless {
|
||||||
|
java {
|
||||||
|
target("src/**/*.java")
|
||||||
|
importOrder()
|
||||||
|
removeUnusedImports()
|
||||||
|
palantirJavaFormat()
|
||||||
|
}
|
||||||
|
|
||||||
|
kotlin {
|
||||||
|
target("src/**/*.kt")
|
||||||
|
ktlint()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-common-conventions")
|
||||||
|
|
||||||
|
`java-library`
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
package buildlogic
|
||||||
|
|
||||||
|
import javax.inject.Inject
|
||||||
|
import org.gradle.api.DefaultTask
|
||||||
|
import org.gradle.api.file.RegularFileProperty
|
||||||
|
import org.gradle.api.provider.ListProperty
|
||||||
|
import org.gradle.api.tasks.Classpath
|
||||||
|
import org.gradle.api.tasks.OutputFile
|
||||||
|
import org.gradle.api.tasks.TaskAction
|
||||||
|
import org.gradle.jvm.toolchain.JavaLanguageVersion
|
||||||
|
import org.gradle.jvm.toolchain.JavaToolchainService
|
||||||
|
import org.gradle.jvm.toolchain.JvmVendorSpec
|
||||||
|
import org.gradle.process.ExecOperations
|
||||||
|
|
||||||
|
abstract class GraalNativeImageTask @Inject constructor(private val toolchains: JavaToolchainService, private val exec: ExecOperations) : DefaultTask() {
|
||||||
|
@get:Classpath abstract val classpath: ListProperty<String>
|
||||||
|
@get:OutputFile abstract val output: RegularFileProperty
|
||||||
|
abstract val mainClass: org.gradle.api.provider.Property<String>
|
||||||
|
|
||||||
|
@TaskAction fun build() {
|
||||||
|
val launcher = toolchains.launcherFor {
|
||||||
|
languageVersion.set(JavaLanguageVersion.of(25))
|
||||||
|
vendor.set(JvmVendorSpec.GRAAL_VM)
|
||||||
|
}.get()
|
||||||
|
val nativeImage = launcher.metadata.installationPath.file("bin/native-image").asFile
|
||||||
|
check(nativeImage.canExecute()) { "The configured Java 25 toolchain must be GraalVM with native-image installed" }
|
||||||
|
output.get().asFile.parentFile.mkdirs()
|
||||||
|
exec.exec { commandLine(nativeImage.absolutePath, "-cp", classpath.get().joinToString(java.io.File.pathSeparator), mainClass.get(), output.get().asFile.absolutePath) }
|
||||||
|
}
|
||||||
|
}
|
||||||
7
gradle.properties
Normal file
7
gradle.properties
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
# This file was generated by the Gradle 'init' task.
|
||||||
|
# https://docs.gradle.org/current/userguide/build_environment.html#sec:gradle_configuration_properties
|
||||||
|
|
||||||
|
org.gradle.configuration-cache=true
|
||||||
|
org.gradle.parallel=true
|
||||||
|
org.gradle.caching=true
|
||||||
|
org.gradle.java.installations.auto-download=true
|
||||||
8
gradle/libs.versions.toml
Normal file
8
gradle/libs.versions.toml
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
# This file was generated by the Gradle 'init' task.
|
||||||
|
# https://docs.gradle.org/current/userguide/version_catalogs.html#sec::toml-dependencies-format
|
||||||
|
|
||||||
|
[versions]
|
||||||
|
kotlin-gradle-plugin = "2.4.0"
|
||||||
|
|
||||||
|
[libraries]
|
||||||
|
kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin-gradle-plugin" }
|
||||||
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
Binary file not shown.
9
gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
9
gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
distributionBase=GRADLE_USER_HOME
|
||||||
|
distributionPath=wrapper/dists
|
||||||
|
distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.1-bin.zip
|
||||||
|
networkTimeout=10000
|
||||||
|
retries=0
|
||||||
|
retryBackOffMs=500
|
||||||
|
validateDistributionUrl=true
|
||||||
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
|
zipStorePath=wrapper/dists
|
||||||
248
gradlew
vendored
Executable file
248
gradlew
vendored
Executable file
|
|
@ -0,0 +1,248 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
#
|
||||||
|
# Copyright © 2015 the original authors.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# gradlew start up script for POSIX generated by Gradle.
|
||||||
|
#
|
||||||
|
# Important for running:
|
||||||
|
#
|
||||||
|
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||||
|
# noncompliant, but you have some other compliant shell such as ksh or
|
||||||
|
# bash, then to run this script, type that shell name before the whole
|
||||||
|
# command line, like:
|
||||||
|
#
|
||||||
|
# ksh gradlew
|
||||||
|
#
|
||||||
|
# Busybox and similar reduced shells will NOT work, because this script
|
||||||
|
# requires all of these POSIX shell features:
|
||||||
|
# * functions;
|
||||||
|
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||||
|
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||||
|
# * compound commands having a testable exit status, especially «case»;
|
||||||
|
# * various built-in commands including «command», «set», and «ulimit».
|
||||||
|
#
|
||||||
|
# Important for patching:
|
||||||
|
#
|
||||||
|
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||||
|
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||||
|
#
|
||||||
|
# The "traditional" practice of packing multiple parameters into a
|
||||||
|
# space-separated string is a well documented source of bugs and security
|
||||||
|
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||||
|
# options in "$@", and eventually passing that to Java.
|
||||||
|
#
|
||||||
|
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||||
|
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||||
|
# see the in-line comments for details.
|
||||||
|
#
|
||||||
|
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||||
|
# Darwin, MinGW, and NonStop.
|
||||||
|
#
|
||||||
|
# (3) This script is generated from the Groovy template
|
||||||
|
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||||
|
# within the Gradle project.
|
||||||
|
#
|
||||||
|
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||||
|
#
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
# Attempt to set APP_HOME
|
||||||
|
|
||||||
|
# Resolve links: $0 may be a link
|
||||||
|
app_path=$0
|
||||||
|
|
||||||
|
# Need this for daisy-chained symlinks.
|
||||||
|
while
|
||||||
|
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||||
|
[ -h "$app_path" ]
|
||||||
|
do
|
||||||
|
ls=$( ls -ld "$app_path" )
|
||||||
|
link=${ls#*' -> '}
|
||||||
|
case $link in #(
|
||||||
|
/*) app_path=$link ;; #(
|
||||||
|
*) app_path=$APP_HOME$link ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# This is normally unused
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
APP_BASE_NAME=${0##*/}
|
||||||
|
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||||
|
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||||
|
|
||||||
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||||
|
MAX_FD=maximum
|
||||||
|
|
||||||
|
warn () {
|
||||||
|
echo "$*"
|
||||||
|
} >&2
|
||||||
|
|
||||||
|
die () {
|
||||||
|
echo
|
||||||
|
echo "$*"
|
||||||
|
echo
|
||||||
|
exit 1
|
||||||
|
} >&2
|
||||||
|
|
||||||
|
# OS specific support (must be 'true' or 'false').
|
||||||
|
cygwin=false
|
||||||
|
msys=false
|
||||||
|
darwin=false
|
||||||
|
nonstop=false
|
||||||
|
case "$( uname )" in #(
|
||||||
|
CYGWIN* ) cygwin=true ;; #(
|
||||||
|
Darwin* ) darwin=true ;; #(
|
||||||
|
MSYS* | MINGW* ) msys=true ;; #(
|
||||||
|
NONSTOP* ) nonstop=true ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Determine the Java command to use to start the JVM.
|
||||||
|
if [ -n "$JAVA_HOME" ] ; then
|
||||||
|
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||||
|
# IBM's JDK on AIX uses strange locations for the executables
|
||||||
|
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||||
|
else
|
||||||
|
JAVACMD=$JAVA_HOME/bin/java
|
||||||
|
fi
|
||||||
|
if [ ! -x "$JAVACMD" ] ; then
|
||||||
|
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||||
|
|
||||||
|
Please set the JAVA_HOME variable in your environment to match the
|
||||||
|
location of your Java installation."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
JAVACMD=java
|
||||||
|
if ! command -v java >/dev/null 2>&1
|
||||||
|
then
|
||||||
|
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||||
|
|
||||||
|
Please set the JAVA_HOME variable in your environment to match the
|
||||||
|
location of your Java installation."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Increase the maximum file descriptors if we can.
|
||||||
|
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||||
|
case $MAX_FD in #(
|
||||||
|
max*)
|
||||||
|
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||||
|
# shellcheck disable=SC2039,SC3045
|
||||||
|
MAX_FD=$( ulimit -H -n ) ||
|
||||||
|
warn "Could not query maximum file descriptor limit"
|
||||||
|
esac
|
||||||
|
case $MAX_FD in #(
|
||||||
|
'' | soft) :;; #(
|
||||||
|
*)
|
||||||
|
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||||
|
# shellcheck disable=SC2039,SC3045
|
||||||
|
ulimit -n "$MAX_FD" ||
|
||||||
|
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Collect all arguments for the java command, stacking in reverse order:
|
||||||
|
# * args from the command line
|
||||||
|
# * the main class name
|
||||||
|
# * -classpath
|
||||||
|
# * -D...appname settings
|
||||||
|
# * --module-path (only if needed)
|
||||||
|
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||||
|
|
||||||
|
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||||
|
if "$cygwin" || "$msys" ; then
|
||||||
|
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||||
|
|
||||||
|
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||||
|
|
||||||
|
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||||
|
for arg do
|
||||||
|
if
|
||||||
|
case $arg in #(
|
||||||
|
-*) false ;; # don't mess with options #(
|
||||||
|
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||||
|
[ -e "$t" ] ;; #(
|
||||||
|
*) false ;;
|
||||||
|
esac
|
||||||
|
then
|
||||||
|
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||||
|
fi
|
||||||
|
# Roll the args list around exactly as many times as the number of
|
||||||
|
# args, so each arg winds up back in the position where it started, but
|
||||||
|
# possibly modified.
|
||||||
|
#
|
||||||
|
# NB: a `for` loop captures its iteration list before it begins, so
|
||||||
|
# changing the positional parameters here affects neither the number of
|
||||||
|
# iterations, nor the values presented in `arg`.
|
||||||
|
shift # remove old arg
|
||||||
|
set -- "$@" "$arg" # push replacement arg
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||||
|
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||||
|
|
||||||
|
# Collect all arguments for the java command:
|
||||||
|
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||||
|
# and any embedded shellness will be escaped.
|
||||||
|
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||||
|
# treated as '${Hostname}' itself on the command line.
|
||||||
|
|
||||||
|
set -- \
|
||||||
|
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||||
|
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||||
|
"$@"
|
||||||
|
|
||||||
|
# Stop when "xargs" is not available.
|
||||||
|
if ! command -v xargs >/dev/null 2>&1
|
||||||
|
then
|
||||||
|
die "xargs is not available"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use "xargs" to parse quoted args.
|
||||||
|
#
|
||||||
|
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||||
|
#
|
||||||
|
# In Bash we could simply go:
|
||||||
|
#
|
||||||
|
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||||
|
# set -- "${ARGS[@]}" "$@"
|
||||||
|
#
|
||||||
|
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||||
|
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||||
|
# character that might be a shell metacharacter, then use eval to reverse
|
||||||
|
# that process (while maintaining the separation between arguments), and wrap
|
||||||
|
# the whole thing up as a single "set" statement.
|
||||||
|
#
|
||||||
|
# This will of course break if any of these variables contains a newline or
|
||||||
|
# an unmatched quote.
|
||||||
|
#
|
||||||
|
|
||||||
|
eval "set -- $(
|
||||||
|
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||||
|
xargs -n1 |
|
||||||
|
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||||
|
tr '\n' ' '
|
||||||
|
)" '"$@"'
|
||||||
|
|
||||||
|
exec "$JAVACMD" "$@"
|
||||||
82
gradlew.bat
vendored
Normal file
82
gradlew.bat
vendored
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
@rem
|
||||||
|
@rem Copyright 2015 the original author or authors.
|
||||||
|
@rem
|
||||||
|
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
@rem you may not use this file except in compliance with the License.
|
||||||
|
@rem You may obtain a copy of the License at
|
||||||
|
@rem
|
||||||
|
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
@rem
|
||||||
|
@rem Unless required by applicable law or agreed to in writing, software
|
||||||
|
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
@rem See the License for the specific language governing permissions and
|
||||||
|
@rem limitations under the License.
|
||||||
|
@rem
|
||||||
|
@rem SPDX-License-Identifier: Apache-2.0
|
||||||
|
@rem
|
||||||
|
|
||||||
|
@if "%DEBUG%"=="" @echo off
|
||||||
|
@rem ##########################################################################
|
||||||
|
@rem
|
||||||
|
@rem gradlew startup script for Windows
|
||||||
|
@rem
|
||||||
|
@rem ##########################################################################
|
||||||
|
|
||||||
|
@rem Set local scope for the variables, and ensure extensions are enabled
|
||||||
|
setlocal EnableExtensions
|
||||||
|
|
||||||
|
set DIRNAME=%~dp0
|
||||||
|
if "%DIRNAME%"=="" set DIRNAME=.
|
||||||
|
@rem This is normally unused
|
||||||
|
set APP_BASE_NAME=%~n0
|
||||||
|
set APP_HOME=%DIRNAME%
|
||||||
|
|
||||||
|
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||||
|
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||||
|
|
||||||
|
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||||
|
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||||
|
|
||||||
|
@rem Find java.exe
|
||||||
|
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||||
|
|
||||||
|
set JAVA_EXE=java.exe
|
||||||
|
%JAVA_EXE% -version >NUL 2>&1
|
||||||
|
if %ERRORLEVEL% equ 0 goto execute
|
||||||
|
|
||||||
|
echo. 1>&2
|
||||||
|
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||||
|
echo. 1>&2
|
||||||
|
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||||
|
echo location of your Java installation. 1>&2
|
||||||
|
|
||||||
|
"%COMSPEC%" /c exit 1
|
||||||
|
|
||||||
|
:findJavaFromJavaHome
|
||||||
|
set JAVA_HOME=%JAVA_HOME:"=%
|
||||||
|
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||||
|
|
||||||
|
if exist "%JAVA_EXE%" goto execute
|
||||||
|
|
||||||
|
echo. 1>&2
|
||||||
|
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||||
|
echo. 1>&2
|
||||||
|
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||||
|
echo location of your Java installation. 1>&2
|
||||||
|
|
||||||
|
"%COMSPEC%" /c exit 1
|
||||||
|
|
||||||
|
:execute
|
||||||
|
@rem Setup the command line
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@rem Execute gradlew
|
||||||
|
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
|
||||||
|
@rem which allows us to clear the local environment before executing the java command
|
||||||
|
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
|
||||||
|
|
||||||
|
:exitWithErrorLevel
|
||||||
|
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
|
||||||
|
"%COMSPEC%" /c exit %ERRORLEVEL%
|
||||||
7
node.yaml.example
Normal file
7
node.yaml.example
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
remote: https://ai.crit.rip/automatedintelligence/geofeeds.git
|
||||||
|
branch: main
|
||||||
|
lfs: true
|
||||||
|
tainted: false
|
||||||
|
passwd: "set-through-secret-manager"
|
||||||
|
bootstrap:
|
||||||
|
- "peer.example.net:7400"
|
||||||
1
packages/build.gradle.kts
Normal file
1
packages/build.gradle.kts
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
plugins { base }
|
||||||
10
packages/decentraland/build.gradle.kts
Normal file
10
packages/decentraland/build.gradle.kts
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-library-conventions")
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
api(project(":packages:serialize"))
|
||||||
|
api(project(":packages:decentraland:kademlia"))
|
||||||
|
api(project(":packages:decentraland:git"))
|
||||||
|
implementation("org.apache.sshd:sshd-core:2.15.0")
|
||||||
|
}
|
||||||
1
packages/decentraland/chksum/build.gradle.kts
Normal file
1
packages/decentraland/chksum/build.gradle.kts
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
plugins { id("buildlogic.kotlin-library-conventions") }
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
package rip.crit.commonip.decentraland.chksum
|
||||||
|
|
||||||
|
import java.security.MessageDigest
|
||||||
|
|
||||||
|
object ContentHash {
|
||||||
|
fun sha256(value: String): String =
|
||||||
|
MessageDigest.getInstance("SHA-256").digest(value.toByteArray()).joinToString("") {
|
||||||
|
"%02x".format(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
3
packages/decentraland/git/build.gradle.kts
Normal file
3
packages/decentraland/git/build.gradle.kts
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
plugins { id("buildlogic.kotlin-library-conventions") }
|
||||||
|
|
||||||
|
dependencies { api(project(":packages:serialize")) }
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
package rip.crit.commonip.decentraland.git
|
||||||
|
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
enum class Taint {
|
||||||
|
CLEAN,
|
||||||
|
TAINTED,
|
||||||
|
}
|
||||||
|
|
||||||
|
data class ProvenancedObservation(
|
||||||
|
val location: GeoLocation,
|
||||||
|
val provider: String,
|
||||||
|
val taint: Taint,
|
||||||
|
val evidence: Set<String>,
|
||||||
|
)
|
||||||
|
|
||||||
|
object PromotionPolicy {
|
||||||
|
fun canPublishToCommunity(value: ProvenancedObservation): Boolean =
|
||||||
|
value.taint == Taint.CLEAN && value.evidence.size >= 2
|
||||||
|
|
||||||
|
fun destination(value: ProvenancedObservation): CommonIpRepository =
|
||||||
|
if (canPublishToCommunity(value)) CommonIpRepositories.community
|
||||||
|
else CommonIpRepositories.taintedFull
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,67 @@
|
||||||
|
package rip.crit.commonip.decentraland.git
|
||||||
|
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
enum class RepositoryTrust {
|
||||||
|
COMMUNITY,
|
||||||
|
TAINTED,
|
||||||
|
}
|
||||||
|
|
||||||
|
data class CommonIpRepository(
|
||||||
|
val name: String,
|
||||||
|
val trust: RepositoryTrust,
|
||||||
|
val graph: RevisionGraph = RevisionGraph(),
|
||||||
|
)
|
||||||
|
|
||||||
|
object CommonIpRepositories {
|
||||||
|
val community = CommonIpRepository("commonip/community", RepositoryTrust.COMMUNITY)
|
||||||
|
val taintedFull = CommonIpRepository("commonip/tainted-full", RepositoryTrust.TAINTED)
|
||||||
|
}
|
||||||
|
|
||||||
|
data class NodeYaml(
|
||||||
|
val remote: String,
|
||||||
|
val branch: String = "main",
|
||||||
|
val lfs: Boolean = true,
|
||||||
|
val tainted: Boolean = false,
|
||||||
|
val passwd: String? = null,
|
||||||
|
val bootstrap: List<String> = emptyList(),
|
||||||
|
)
|
||||||
|
|
||||||
|
object NodeYamlReader {
|
||||||
|
fun parse(text: String): NodeYaml {
|
||||||
|
val values =
|
||||||
|
text
|
||||||
|
.lineSequence()
|
||||||
|
.map(String::trim)
|
||||||
|
.filter { ':' in it && !it.startsWith('#') }
|
||||||
|
.associate {
|
||||||
|
it.substringBefore(':').trim() to it.substringAfter(':').trim().trim('"', '\'')
|
||||||
|
}
|
||||||
|
val bootstrap = text.lineSequence().map(String::trim).filter { it.startsWith("-") }.map { it.removePrefix("-").trim().trim('"', '\'') }.toList()
|
||||||
|
return NodeYaml(
|
||||||
|
values["remote"] ?: error("node.yaml requires remote"),
|
||||||
|
values["branch"] ?: "main",
|
||||||
|
values["lfs"]?.toBooleanStrictOrNull() ?: true,
|
||||||
|
values["tainted"]?.toBooleanStrictOrNull() ?: false,
|
||||||
|
values["passwd"],
|
||||||
|
bootstrap,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class GitLfsSync(private val repository: Path) {
|
||||||
|
fun sync(config: NodeYaml): Boolean =
|
||||||
|
runCatching {
|
||||||
|
fun command(vararg args: String) =
|
||||||
|
ProcessBuilder(*args)
|
||||||
|
.directory(repository.toFile())
|
||||||
|
.inheritIO()
|
||||||
|
.start()
|
||||||
|
.waitFor() == 0
|
||||||
|
command("git", "remote", "get-url", "origin") ||
|
||||||
|
command("git", "remote", "add", "origin", config.remote)
|
||||||
|
command("git", "fetch", "origin", config.branch) &&
|
||||||
|
(!config.lfs || command("git", "lfs", "pull", "origin", config.branch))
|
||||||
|
}
|
||||||
|
.getOrDefault(false)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,99 @@
|
||||||
|
package rip.crit.commonip.decentraland.git
|
||||||
|
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.UUID
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
data class GeoCommit(
|
||||||
|
val id: String = UUID.randomUUID().toString(),
|
||||||
|
val parents: Set<String>,
|
||||||
|
val author: String,
|
||||||
|
val observation: GeoLocation,
|
||||||
|
val createdAt: Instant = Instant.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
enum class MergeState {
|
||||||
|
FAST_FORWARD,
|
||||||
|
MERGED,
|
||||||
|
CONFLICT,
|
||||||
|
}
|
||||||
|
|
||||||
|
data class MergeResult(
|
||||||
|
val state: MergeState,
|
||||||
|
val commit: GeoCommit?,
|
||||||
|
val competing: List<GeoCommit>,
|
||||||
|
val message: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
class RevisionGraph {
|
||||||
|
private val commits = linkedMapOf<String, GeoCommit>()
|
||||||
|
|
||||||
|
fun import(commit: GeoCommit): Boolean {
|
||||||
|
if (commit.parents.any { it !in commits }) return false
|
||||||
|
commits.putIfAbsent(commit.id, commit)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
fun commit(author: String, observation: GeoLocation, parents: Set<String> = heads()): GeoCommit =
|
||||||
|
GeoCommit(parents = parents, author = author, observation = observation).also {
|
||||||
|
commits[it.id] = it
|
||||||
|
}
|
||||||
|
|
||||||
|
fun heads(): Set<String> = commits.keys - commits.values.flatMap { it.parents }.toSet()
|
||||||
|
|
||||||
|
fun merge(left: String, right: String, author: String): MergeResult {
|
||||||
|
val a =
|
||||||
|
commits[left]
|
||||||
|
?: return MergeResult(
|
||||||
|
MergeState.CONFLICT,
|
||||||
|
null,
|
||||||
|
emptyList(),
|
||||||
|
"left revision is unknown",
|
||||||
|
)
|
||||||
|
val b =
|
||||||
|
commits[right]
|
||||||
|
?: return MergeResult(
|
||||||
|
MergeState.CONFLICT,
|
||||||
|
null,
|
||||||
|
emptyList(),
|
||||||
|
"right revision is unknown",
|
||||||
|
)
|
||||||
|
if (isAncestor(a.id, b.id))
|
||||||
|
return MergeResult(MergeState.FAST_FORWARD, b, emptyList(), "left fast-forwards to right")
|
||||||
|
if (isAncestor(b.id, a.id))
|
||||||
|
return MergeResult(MergeState.FAST_FORWARD, a, emptyList(), "right fast-forwards to left")
|
||||||
|
if (!compatible(a.observation, b.observation))
|
||||||
|
return MergeResult(
|
||||||
|
MergeState.CONFLICT,
|
||||||
|
null,
|
||||||
|
listOf(a, b),
|
||||||
|
"country or coordinates disagree, proof-of-location required",
|
||||||
|
)
|
||||||
|
val winner = listOf(a, b).maxBy { it.observation.confidence }
|
||||||
|
val merged =
|
||||||
|
GeoCommit(
|
||||||
|
parents = setOf(a.id, b.id),
|
||||||
|
author = author,
|
||||||
|
observation =
|
||||||
|
winner.observation.copy(
|
||||||
|
source = "merge:$author",
|
||||||
|
confidence = (a.observation.confidence + b.observation.confidence) / 2,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
commits[merged.id] = merged
|
||||||
|
return MergeResult(MergeState.MERGED, merged, emptyList(), "two verified revisions merged")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun isAncestor(ancestor: String, descendant: String): Boolean =
|
||||||
|
commits[descendant]?.parents?.let { parents ->
|
||||||
|
ancestor in parents || parents.any { isAncestor(ancestor, it) }
|
||||||
|
} ?: false
|
||||||
|
|
||||||
|
private fun compatible(a: GeoLocation, b: GeoLocation): Boolean {
|
||||||
|
val aLat = a.latitude
|
||||||
|
val bLat = b.latitude
|
||||||
|
return a.prefix == b.prefix &&
|
||||||
|
a.countryCode == b.countryCode &&
|
||||||
|
(aLat == null || bLat == null || kotlin.math.abs(aLat - bLat) < 1.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
package rip.crit.commonip.decentraland.git
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets
|
||||||
|
import java.security.PrivateKey
|
||||||
|
import java.security.PublicKey
|
||||||
|
import java.security.Signature
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.Base64
|
||||||
|
|
||||||
|
data class SignedGeoCommit(val revision: GeoCommit, val signature: String) {
|
||||||
|
fun payload(): ByteArray =
|
||||||
|
listOf(
|
||||||
|
revision.id,
|
||||||
|
revision.parents.sorted().joinToString(","),
|
||||||
|
revision.author,
|
||||||
|
revision.observation.prefix,
|
||||||
|
revision.observation.countryCode,
|
||||||
|
revision.observation.latitude,
|
||||||
|
revision.observation.longitude,
|
||||||
|
revision.createdAt,
|
||||||
|
)
|
||||||
|
.joinToString("|")
|
||||||
|
.toByteArray(StandardCharsets.UTF_8)
|
||||||
|
}
|
||||||
|
|
||||||
|
object CommitSigner {
|
||||||
|
fun sign(revision: GeoCommit, key: PrivateKey): SignedGeoCommit {
|
||||||
|
val unsigned = SignedGeoCommit(revision, "")
|
||||||
|
val signature =
|
||||||
|
Signature.getInstance("Ed25519")
|
||||||
|
.apply {
|
||||||
|
initSign(key)
|
||||||
|
update(unsigned.payload())
|
||||||
|
}
|
||||||
|
.sign()
|
||||||
|
return unsigned.copy(signature = Base64.getEncoder().encodeToString(signature))
|
||||||
|
}
|
||||||
|
|
||||||
|
fun verify(commit: SignedGeoCommit, key: PublicKey): Boolean =
|
||||||
|
runCatching {
|
||||||
|
Signature.getInstance("Ed25519")
|
||||||
|
.apply {
|
||||||
|
initVerify(key)
|
||||||
|
update(commit.payload())
|
||||||
|
}
|
||||||
|
.verify(Base64.getDecoder().decode(commit.signature))
|
||||||
|
}
|
||||||
|
.getOrDefault(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
data class ProofOfLocation(
|
||||||
|
val revisionId: String,
|
||||||
|
val observer: String,
|
||||||
|
val observedAt: Instant,
|
||||||
|
val reachable: Boolean,
|
||||||
|
val latencyMs: Long?,
|
||||||
|
val signature: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
class SignedCommitLedger(
|
||||||
|
private val graph: RevisionGraph,
|
||||||
|
private val keys: Map<String, PublicKey>,
|
||||||
|
) {
|
||||||
|
fun accept(commit: SignedGeoCommit): Boolean =
|
||||||
|
(keys[commit.revision.author]?.let { CommitSigner.verify(commit, it) } ?: false) &&
|
||||||
|
graph.import(commit.revision)
|
||||||
|
|
||||||
|
fun resolve(left: SignedGeoCommit, right: SignedGeoCommit, author: String): MergeResult {
|
||||||
|
require(accept(left) && accept(right)) { "all merge inputs must have valid signatures" }
|
||||||
|
return graph.merge(left.revision.id, right.revision.id, author)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
package rip.crit.commonip.decentraland.git
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertFalse
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
class RepositoryTest {
|
||||||
|
@Test
|
||||||
|
fun `node yaml retains trust and password fields`() {
|
||||||
|
val config = NodeYamlReader.parse("remote: ssh://example/repo\ntainted: true\npasswd: secret\nbootstrap:\n - peer.example:7400")
|
||||||
|
assertTrue(config.tainted)
|
||||||
|
assertEquals("secret", config.passwd)
|
||||||
|
assertEquals(listOf("peer.example:7400"), config.bootstrap)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `community promotion needs clean independent evidence`() {
|
||||||
|
val location = GeoLocation("192.0.2.0/24", "US")
|
||||||
|
assertFalse(
|
||||||
|
PromotionPolicy.canPublishToCommunity(
|
||||||
|
ProvenancedObservation(location, "test", Taint.CLEAN, setOf("one"))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertTrue(
|
||||||
|
PromotionPolicy.canPublishToCommunity(
|
||||||
|
ProvenancedObservation(location, "test", Taint.CLEAN, setOf("one", "two"))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
7
packages/decentraland/kademlia/build.gradle.kts
Normal file
7
packages/decentraland/kademlia/build.gradle.kts
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
plugins { id("buildlogic.kotlin-library-conventions") }
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
api(project(":packages:serialize"))
|
||||||
|
implementation("io.ep2p:kademlia-api:5.1.2-RELEASE")
|
||||||
|
implementation("io.ep2p:kademlia-netty:0.3.2-RELEASE")
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,111 @@
|
||||||
|
package rip.crit.commonip.decentraland.kademlia
|
||||||
|
|
||||||
|
import java.net.DatagramPacket
|
||||||
|
import java.net.DatagramSocket
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.security.MessageDigest
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import kotlin.concurrent.thread
|
||||||
|
|
||||||
|
data class KademliaPeer(val id: String, val address: InetSocketAddress)
|
||||||
|
|
||||||
|
class KademliaNode(private val bind: InetSocketAddress, private val registry: NodeRegistry) :
|
||||||
|
AutoCloseable {
|
||||||
|
val id = sha256("${bind.hostString}:${bind.port}")
|
||||||
|
private val namer = NodeNamer()
|
||||||
|
private val peers = ConcurrentHashMap<String, KademliaPeer>()
|
||||||
|
private val values = ConcurrentHashMap<String, String>()
|
||||||
|
private val socket = DatagramSocket(bind)
|
||||||
|
private val lastOutbound = ConcurrentHashMap<String, Long>()
|
||||||
|
@Volatile private var running = true
|
||||||
|
|
||||||
|
fun start() =
|
||||||
|
thread(name = "commonip-kademlia-$id", isDaemon = true) {
|
||||||
|
registry.register(
|
||||||
|
NetworkNode(
|
||||||
|
namer.name(id),
|
||||||
|
socket.localSocketAddress.toString(),
|
||||||
|
NodeState.ONLINE,
|
||||||
|
0,
|
||||||
|
values.size,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
val bytes = ByteArray(8192)
|
||||||
|
while (running) runCatching {
|
||||||
|
val packet = DatagramPacket(bytes, bytes.size)
|
||||||
|
socket.receive(packet)
|
||||||
|
handle(packet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun bootstrap(seed: InetSocketAddress) {
|
||||||
|
if (peers.values.none { it.address == seed }) send(seed, "PING|$id|${bind.hostString}|${socket.localPort}")
|
||||||
|
}
|
||||||
|
|
||||||
|
fun store(key: String, value: String) {
|
||||||
|
values[key] = value
|
||||||
|
closest(key).forEach { send(it.address, "STORE|$id|$key|${value.encodeBase64()}") }
|
||||||
|
}
|
||||||
|
|
||||||
|
fun findValue(key: String): String? = values[key]
|
||||||
|
|
||||||
|
fun closest(key: String, count: Int = 20): List<KademliaPeer> =
|
||||||
|
peers.values.sortedBy { xorDistance(it.id, key) }.take(count)
|
||||||
|
|
||||||
|
private fun handle(packet: DatagramPacket) {
|
||||||
|
val data =
|
||||||
|
packet.data
|
||||||
|
.copyOfRange(packet.offset, packet.offset + packet.length)
|
||||||
|
.decodeToString()
|
||||||
|
.split('|', limit = 5)
|
||||||
|
val remote = InetSocketAddress(packet.address, packet.port)
|
||||||
|
when (data.firstOrNull()) {
|
||||||
|
"PING" -> {
|
||||||
|
peers[data[1]] = KademliaPeer(data[1], InetSocketAddress(data[2], data[3].toInt()))
|
||||||
|
val name = namer.name(data[1])
|
||||||
|
registry.register(NetworkNode(name, remote.toString(), NodeState.ONLINE, null, 0))
|
||||||
|
registry.publish(name, "kademlia ping", remote.toString())
|
||||||
|
send(remote, "PONG|$id")
|
||||||
|
}
|
||||||
|
"PONG" -> {
|
||||||
|
peers[data[1]] = KademliaPeer(data[1], remote)
|
||||||
|
val name = namer.name(data[1])
|
||||||
|
registry.register(NetworkNode(name, remote.toString(), NodeState.ONLINE, null, 0))
|
||||||
|
registry.publish(name, "kademlia bootstrap confirmed", remote.toString())
|
||||||
|
}
|
||||||
|
"STORE" -> values[data[2]] = data[3].decodeBase64()
|
||||||
|
"FIND_VALUE" ->
|
||||||
|
values[data[2]]?.let { send(remote, "VALUE|$id|${data[2]}|${it.encodeBase64()}") }
|
||||||
|
"FIND_NODE" ->
|
||||||
|
send(
|
||||||
|
remote,
|
||||||
|
"NODES|$id|${closest(data[2]).joinToString(";") { "${it.id}@${it.address.hostString}:${it.address.port}" }}",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun send(target: InetSocketAddress, message: String) {
|
||||||
|
val key = "${target.hostString}:${target.port}:${message.substringBefore('|')}"
|
||||||
|
val previous = lastOutbound.put(key, System.currentTimeMillis()) ?: 0
|
||||||
|
if (System.currentTimeMillis() - previous < 500) return
|
||||||
|
socket.send(DatagramPacket(message.encodeToByteArray(), message.length, target))
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
running = false
|
||||||
|
socket.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun sha256(value: String) =
|
||||||
|
MessageDigest.getInstance("SHA-256").digest(value.encodeToByteArray()).joinToString("") {
|
||||||
|
"%02x".format(it)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun xorDistance(left: String, right: String) =
|
||||||
|
left.zip(right.padEnd(left.length, '0')).count { it.first != it.second }
|
||||||
|
|
||||||
|
private fun String.encodeBase64() =
|
||||||
|
java.util.Base64.getUrlEncoder().encodeToString(encodeToByteArray())
|
||||||
|
|
||||||
|
private fun String.decodeBase64() = java.util.Base64.getUrlDecoder().decode(this).decodeToString()
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
package rip.crit.commonip.decentraland.kademlia
|
||||||
|
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
|
||||||
|
class NodeNamer {
|
||||||
|
private val votes = ConcurrentHashMap<String, ConcurrentHashMap<String, MutableSet<String>>>()
|
||||||
|
|
||||||
|
fun observe(nodeId: String, observerId: String, alias: String) {
|
||||||
|
votes
|
||||||
|
.getOrPut(nodeId) { ConcurrentHashMap() }
|
||||||
|
.getOrPut(alias) { ConcurrentHashMap.newKeySet() }
|
||||||
|
.add(observerId)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun name(nodeId: String): String =
|
||||||
|
votes[nodeId]?.maxByOrNull { it.value.size }?.takeIf { it.value.size >= 2 }?.key
|
||||||
|
?: deterministic(nodeId)
|
||||||
|
|
||||||
|
private fun deterministic(id: String): String {
|
||||||
|
val adjective = arrayOf("amber", "cedar", "delta", "ember", "flint", "grove", "harbor", "iris")
|
||||||
|
val noun = arrayOf("atlas", "beacon", "cinder", "drift", "echo", "forge", "glade", "horizon")
|
||||||
|
val number = id.take(8).toLong(16)
|
||||||
|
return "${adjective[(number % adjective.size).toInt()]}-${noun[((number / adjective.size) % noun.size).toInt()]}-${id.takeLast(4)}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
package rip.crit.commonip.decentraland.kademlia
|
||||||
|
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
|
||||||
|
enum class NodeState {
|
||||||
|
ONLINE,
|
||||||
|
DEGRADED,
|
||||||
|
OFFLINE,
|
||||||
|
}
|
||||||
|
|
||||||
|
data class NetworkNode(
|
||||||
|
val id: String,
|
||||||
|
val endpoint: String,
|
||||||
|
val state: NodeState,
|
||||||
|
val latencyMs: Long?,
|
||||||
|
val records: Int,
|
||||||
|
val lastSeen: Instant = Instant.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
data class NetworkActivity(
|
||||||
|
val at: Instant,
|
||||||
|
val nodeId: String,
|
||||||
|
val action: String,
|
||||||
|
val detail: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
class NodeRegistry {
|
||||||
|
private val nodes = linkedMapOf<String, NetworkNode>()
|
||||||
|
private val activity = CopyOnWriteArrayList<NetworkActivity>()
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
fun register(node: NetworkNode) {
|
||||||
|
nodes[node.id] = node
|
||||||
|
publish(node.id, "peer announced", "${node.endpoint} is ${node.state.name.lowercase()}")
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
fun update(id: String, state: NodeState, latencyMs: Long? = null, records: Int? = null) {
|
||||||
|
val prior = nodes[id] ?: return
|
||||||
|
nodes[id] =
|
||||||
|
prior.copy(
|
||||||
|
state = state,
|
||||||
|
latencyMs = latencyMs ?: prior.latencyMs,
|
||||||
|
records = records ?: prior.records,
|
||||||
|
lastSeen = Instant.now(),
|
||||||
|
)
|
||||||
|
publish(id, "status changed", state.name.lowercase())
|
||||||
|
}
|
||||||
|
|
||||||
|
fun snapshot(): List<NetworkNode> = synchronized(this) { nodes.values.sortedBy { it.id } }
|
||||||
|
|
||||||
|
fun feed(limit: Int = 12): List<NetworkActivity> = activity.takeLast(limit).reversed()
|
||||||
|
|
||||||
|
fun publish(nodeId: String, action: String, detail: String) {
|
||||||
|
activity += NetworkActivity(Instant.now(), nodeId, action, detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
package rip.crit.commonip.decentraland.kademlia
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class NodeNamerTest {
|
||||||
|
@Test
|
||||||
|
fun `two peer votes override deterministic node name`() {
|
||||||
|
val namer = NodeNamer()
|
||||||
|
namer.observe("abcdef012345", "peer-a", "north")
|
||||||
|
namer.observe("abcdef012345", "peer-b", "north")
|
||||||
|
assertEquals("north", namer.name("abcdef012345"))
|
||||||
|
}
|
||||||
|
}
|
||||||
3
packages/decentraland/settings.gradle.kts
Normal file
3
packages/decentraland/settings.gradle.kts
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
rootProject.name = "commonip-decentraland"
|
||||||
|
|
||||||
|
include("kademlia", "git", "chksum")
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
package rip.crit.commonip.decentraland
|
||||||
|
|
||||||
|
import java.nio.file.Path
|
||||||
|
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory
|
||||||
|
import org.apache.sshd.server.SshServer
|
||||||
|
import org.apache.sshd.server.auth.password.PasswordAuthenticator
|
||||||
|
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider
|
||||||
|
|
||||||
|
class SshDashboardService(
|
||||||
|
port: Int,
|
||||||
|
hostKey: Path,
|
||||||
|
username: String,
|
||||||
|
password: CharArray,
|
||||||
|
private val dashboard: () -> String,
|
||||||
|
) : AutoCloseable {
|
||||||
|
private val server =
|
||||||
|
SshServer.setUpDefaultServer().apply {
|
||||||
|
this.port = port
|
||||||
|
keyPairProvider = SimpleGeneratorHostKeyProvider(hostKey)
|
||||||
|
passwordAuthenticator = PasswordAuthenticator { user, candidate, _ ->
|
||||||
|
user == username && candidate.toCharArray().contentEquals(password)
|
||||||
|
}
|
||||||
|
fileSystemFactory = VirtualFileSystemFactory()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun start() {
|
||||||
|
server.start()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun render(): String = dashboard()
|
||||||
|
|
||||||
|
override fun close() = server.close()
|
||||||
|
}
|
||||||
13
packages/ml/build.gradle.kts
Normal file
13
packages/ml/build.gradle.kts
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-library-conventions")
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
api(project(":packages:serialize"))
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,235 @@
|
||||||
|
package rip.crit.commonip.ml
|
||||||
|
|
||||||
|
import kotlin.math.*
|
||||||
|
|
||||||
|
data class GeoLocation(
|
||||||
|
val latitude: Double?,
|
||||||
|
val longitude: Double?,
|
||||||
|
val confidence: Double = 1.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class LocationEstimate(
|
||||||
|
val latitude: Double,
|
||||||
|
val longitude: Double,
|
||||||
|
val confidence: Double,
|
||||||
|
val sampleCount: Int,
|
||||||
|
val discardedCount: Int,
|
||||||
|
val spatialSpreadKm: Double,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class Vector3D(val x: Double, val y: Double, val z: Double) {
|
||||||
|
fun normalize(): Vector3D {
|
||||||
|
val mag = sqrt(x * x + y * y + z * z)
|
||||||
|
return if (mag == 0.0) Vector3D(0.0, 0.0, 0.0) else Vector3D(x / mag, y / mag, z / mag)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun toGeoLocation(confidence: Double = 1.0): GeoLocation {
|
||||||
|
val norm = this.normalize()
|
||||||
|
val lat = Math.toDegrees(asin(norm.z.coerceIn(-1.0, 1.0)))
|
||||||
|
val lon = Math.toDegrees(atan2(norm.y, norm.x))
|
||||||
|
return GeoLocation(lat, lon, confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun GeoLocation.toVector3D(): Vector3D {
|
||||||
|
val latRad = Math.toRadians(this.latitude!!)
|
||||||
|
val lonRad = Math.toRadians(this.longitude!!)
|
||||||
|
return Vector3D(
|
||||||
|
x = cos(latRad) * cos(lonRad),
|
||||||
|
y = cos(latRad) * sin(lonRad),
|
||||||
|
z = sin(latRad),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
class LocationEstimator(
|
||||||
|
private val dbscanEpsKm: Double = 250.0,
|
||||||
|
private val dbscanMinPts: Int = 2,
|
||||||
|
private val medianIterations: Int = 20,
|
||||||
|
) {
|
||||||
|
|
||||||
|
fun estimate(observations: List<GeoLocation>): LocationEstimate? {
|
||||||
|
val validPoints = observations.filter { it.latitude != null && it.longitude != null }
|
||||||
|
if (validPoints.isEmpty()) return null
|
||||||
|
|
||||||
|
val clusteredPoints = dbscanCluster(validPoints, dbscanEpsKm, dbscanMinPts)
|
||||||
|
val selected = clusteredPoints.ifEmpty { validPoints }
|
||||||
|
|
||||||
|
val (estLat, estLon) = computeGeometricMedian(selected, medianIterations)
|
||||||
|
|
||||||
|
val distances = selected.map { distanceKm(estLat, estLon, it.latitude!!, it.longitude!!) }
|
||||||
|
val standardDistanceKm = sqrt(distances.sumOf { it * it } / selected.size)
|
||||||
|
|
||||||
|
val meanSampleConfidence = selected.map { max(it.confidence, 0.01) }.average()
|
||||||
|
val densityFactor = 1.0 / (1.0 + exp(-(selected.size - 3.0) / 2.0))
|
||||||
|
val spreadPenalty = 1.0 / (1.0 + (standardDistanceKm / 300.0))
|
||||||
|
|
||||||
|
val overallConfidence =
|
||||||
|
(meanSampleConfidence * densityFactor * spreadPenalty).coerceIn(0.0, 1.0)
|
||||||
|
|
||||||
|
return LocationEstimate(
|
||||||
|
latitude = estLat,
|
||||||
|
longitude = estLon,
|
||||||
|
confidence = overallConfidence,
|
||||||
|
sampleCount = selected.size,
|
||||||
|
discardedCount = validPoints.size - selected.size,
|
||||||
|
spatialSpreadKm = standardDistanceKm,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Finds the primary spatial cluster using DBSCAN logic. */
|
||||||
|
private fun dbscanCluster(
|
||||||
|
points: List<GeoLocation>,
|
||||||
|
epsKm: Double,
|
||||||
|
minPts: Int,
|
||||||
|
): List<GeoLocation> {
|
||||||
|
val visited = BooleanArray(points.size)
|
||||||
|
val clusters = mutableListOf<MutableList<GeoLocation>>()
|
||||||
|
|
||||||
|
for (i in points.indices) {
|
||||||
|
if (visited[i]) continue
|
||||||
|
visited[i] = true
|
||||||
|
|
||||||
|
val neighbors = getNeighbors(points, i, epsKm)
|
||||||
|
if (neighbors.size >= minPts) {
|
||||||
|
val currentCluster = mutableListOf(points[i])
|
||||||
|
val queue = neighbors.toMutableList()
|
||||||
|
var qIndex = 0
|
||||||
|
|
||||||
|
while (qIndex < queue.size) {
|
||||||
|
val pIndex = queue[qIndex++]
|
||||||
|
if (!visited[pIndex]) {
|
||||||
|
visited[pIndex] = true
|
||||||
|
val subNeighbors = getNeighbors(points, pIndex, epsKm)
|
||||||
|
if (subNeighbors.size >= minPts) {
|
||||||
|
queue.addAll(subNeighbors.filter { !queue.contains(it) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!currentCluster.contains(points[pIndex])) {
|
||||||
|
currentCluster.add(points[pIndex])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clusters.add(currentCluster)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return clusters.maxByOrNull { cluster -> cluster.sumOf { max(it.confidence, 0.05) } }
|
||||||
|
?: emptyList()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getNeighbors(points: List<GeoLocation>, centerIdx: Int, epsKm: Double): List<Int> {
|
||||||
|
val center = points[centerIdx]
|
||||||
|
val result = mutableListOf<Int>()
|
||||||
|
for (i in points.indices) {
|
||||||
|
if (
|
||||||
|
distanceKm(
|
||||||
|
center.latitude!!,
|
||||||
|
center.longitude!!,
|
||||||
|
points[i].latitude!!,
|
||||||
|
points[i].longitude!!,
|
||||||
|
) <= epsKm
|
||||||
|
) {
|
||||||
|
result.add(i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Weiszfeld's algorithm for finding the spatial geometric median (L1 median). */
|
||||||
|
private fun computeGeometricMedian(
|
||||||
|
points: List<GeoLocation>,
|
||||||
|
iterations: Int,
|
||||||
|
): Pair<Double, Double> {
|
||||||
|
if (points.size == 1) return Pair(points[0].latitude!!, points[0].longitude!!)
|
||||||
|
|
||||||
|
val weights = points.map { max(it.confidence, 0.05) }
|
||||||
|
val sumWeight = weights.sum()
|
||||||
|
|
||||||
|
var currentVector =
|
||||||
|
points
|
||||||
|
.zip(weights)
|
||||||
|
.map { (pt, w) ->
|
||||||
|
val vec = pt.toVector3D()
|
||||||
|
Vector3D(vec.x * w, vec.y * w, vec.z * w)
|
||||||
|
}
|
||||||
|
.reduce { acc, v ->
|
||||||
|
Vector3D(acc.x + v.x, acc.y + v.y, acc.z + v.z)
|
||||||
|
}
|
||||||
|
.let { Vector3D(it.x / sumWeight, it.y / sumWeight, it.z / sumWeight).normalize() }
|
||||||
|
|
||||||
|
var currentGeo = currentVector.toGeoLocation()
|
||||||
|
|
||||||
|
repeat(iterations) {
|
||||||
|
var sumX = 0.0
|
||||||
|
var sumY = 0.0
|
||||||
|
var sumZ = 0.0
|
||||||
|
var totalW = 0.0
|
||||||
|
|
||||||
|
for (i in points.indices) {
|
||||||
|
val ptVec = points[i].toVector3D()
|
||||||
|
val dist =
|
||||||
|
max(
|
||||||
|
distanceKm(
|
||||||
|
currentGeo.latitude!!,
|
||||||
|
currentGeo.longitude!!,
|
||||||
|
points[i].latitude!!,
|
||||||
|
points[i].longitude!!,
|
||||||
|
),
|
||||||
|
0.1,
|
||||||
|
)
|
||||||
|
val invWeight = weights[i] / dist
|
||||||
|
|
||||||
|
sumX += ptVec.x * invWeight
|
||||||
|
sumY += ptVec.y * invWeight
|
||||||
|
sumZ += ptVec.z * invWeight
|
||||||
|
totalW += invWeight
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalW > 0.0) {
|
||||||
|
currentVector = Vector3D(sumX / totalW, sumY / totalW, sumZ / totalW).normalize()
|
||||||
|
currentGeo = currentVector.toGeoLocation()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Pair(currentGeo.latitude!!, currentGeo.longitude!!)
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
fun distanceKm(aLat: Double, aLon: Double, bLat: Double, bLon: Double): Double {
|
||||||
|
val dLat = Math.toRadians(bLat - aLat)
|
||||||
|
val dLon = Math.toRadians(bLon - aLon)
|
||||||
|
val h =
|
||||||
|
sin(dLat / 2).pow(2) +
|
||||||
|
cos(Math.toRadians(aLat)) * cos(Math.toRadians(bLat)) * sin(dLon / 2).pow(2)
|
||||||
|
return 6371.0 * 2 * atan2(sqrt(h), sqrt(1 - h))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data class NodeTarget(
|
||||||
|
val nodeId: String,
|
||||||
|
val capacity: Int,
|
||||||
|
val latitude: Double?,
|
||||||
|
val longitude: Double?,
|
||||||
|
)
|
||||||
|
|
||||||
|
object LocationDispatcher {
|
||||||
|
fun select(
|
||||||
|
nodes: List<NodeTarget>,
|
||||||
|
estimate: LocationEstimate,
|
||||||
|
count: Int = 3,
|
||||||
|
): List<NodeTarget> =
|
||||||
|
nodes
|
||||||
|
.filter { it.capacity > 0 && it.latitude != null && it.longitude != null }
|
||||||
|
.sortedBy { node ->
|
||||||
|
val distance =
|
||||||
|
LocationEstimator.distanceKm(
|
||||||
|
estimate.latitude,
|
||||||
|
estimate.longitude,
|
||||||
|
node.latitude!!,
|
||||||
|
node.longitude!!,
|
||||||
|
)
|
||||||
|
|
||||||
|
distance / sqrt(node.capacity.toDouble())
|
||||||
|
}
|
||||||
|
.take(count)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
package rip.crit.commonip.ml
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class LocationEstimatorTest {
|
||||||
|
@Test
|
||||||
|
fun `estimator rejects distant outlier`() {
|
||||||
|
val result =
|
||||||
|
LocationEstimator()
|
||||||
|
.estimate(
|
||||||
|
listOf(
|
||||||
|
GeoLocation(40.0, -75.0, 0.9),
|
||||||
|
GeoLocation(40.1, -75.1, 0.9),
|
||||||
|
GeoLocation(-33.0, 151.0, 0.9),
|
||||||
|
)
|
||||||
|
)!!
|
||||||
|
assertEquals(1, result.discardedCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
13
packages/rawr/build.gradle.kts
Normal file
13
packages/rawr/build.gradle.kts
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-library-conventions")
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
api(project(":packages:serialize"))
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
package rip.crit.commonip.rawr.feeds
|
||||||
|
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||||
|
|
||||||
|
object RFC8805 {
|
||||||
|
fun parse(csv: String): GeoFeed = GeoFeedCodec.decode(csv, GeoFeedFormat.CSV)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
package rip.crit.commonip.rawr.feeds
|
||||||
|
|
||||||
|
data class RFC9632(
|
||||||
|
val feedUrl: String,
|
||||||
|
val signatureUrl: String?,
|
||||||
|
val valid: Boolean,
|
||||||
|
val signer: String? = null,
|
||||||
|
) {
|
||||||
|
companion object {
|
||||||
|
fun fromDnsRecord(record: String): RFC9632? {
|
||||||
|
val values =
|
||||||
|
record
|
||||||
|
.split(';')
|
||||||
|
.map { it.trim() }
|
||||||
|
.associate { it.substringBefore('=').lowercase() to it.substringAfter('=', "") }
|
||||||
|
val url = values["geofeed"] ?: return null
|
||||||
|
return RFC9632(url, values["sig"], values["valid"] != "false", values["signer"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
package rip.crit.commonip.rawr.feeds
|
||||||
|
|
||||||
|
import rip.crit.commonip.rawr.types.Ingest
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
class RirFeed(private val name: String, private val loader: () -> String) : Ingest {
|
||||||
|
override fun ingest(): GeoFeed =
|
||||||
|
GeoFeedCodec.decode(loader(), GeoFeedFormat.CSV).let { feed ->
|
||||||
|
feed.copy(records = feed.records.map { it.copy(source = name) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
object DelegatedStats {
|
||||||
|
fun parse(payload: String, source: String): GeoFeed =
|
||||||
|
GeoFeed(
|
||||||
|
payload
|
||||||
|
.lineSequence()
|
||||||
|
.filter { it.isNotBlank() && !it.startsWith('#') && !it.startsWith("2|") }
|
||||||
|
.mapNotNull { line ->
|
||||||
|
val f = line.split('|')
|
||||||
|
if (f.size < 5 || f[2].lowercase() !in setOf("ipv4", "ipv6") || f[1] == "*") null
|
||||||
|
else {
|
||||||
|
val prefix =
|
||||||
|
if (f[2].lowercase() == "ipv4") "${f[3]}/${prefixFromCount(f[4])}"
|
||||||
|
else "${f[3]}/${f.getOrElse(4) { "48" }}"
|
||||||
|
GeoLocation(prefix, f[1].uppercase(), source = source, confidence = 0.25)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.toList()
|
||||||
|
)
|
||||||
|
|
||||||
|
private fun prefixFromCount(count: String): Int =
|
||||||
|
32 - Integer.numberOfTrailingZeros(count.toIntOrNull() ?: 1)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
class AFRINIC(loader: (() -> String)? = null) :
|
||||||
|
RirProvider(
|
||||||
|
"AFRINIC",
|
||||||
|
"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
|
||||||
|
loader,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
class APNIC(loader: (() -> String)? = null) :
|
||||||
|
RirProvider(
|
||||||
|
"APNIC",
|
||||||
|
"https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest",
|
||||||
|
loader,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
class ARINIngest(loader: (() -> String)? = null) :
|
||||||
|
RirProvider(
|
||||||
|
"ARIN",
|
||||||
|
"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
|
||||||
|
loader,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
class LACNIC(loader: (() -> String)? = null) :
|
||||||
|
RirProvider(
|
||||||
|
"LACNIC",
|
||||||
|
"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
|
||||||
|
loader,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
class RIPENCC(loader: (() -> String)? = null) :
|
||||||
|
RirProvider(
|
||||||
|
"RIPE NCC",
|
||||||
|
"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
|
||||||
|
loader,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
import java.net.URI
|
||||||
|
import java.net.http.HttpClient
|
||||||
|
import java.net.http.HttpRequest
|
||||||
|
import java.net.http.HttpResponse
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||||
|
|
||||||
|
enum class RirMetadataField(val keys: Set<String>) {
|
||||||
|
REMARKS(setOf("remarks", "remark")),
|
||||||
|
COMMENT(setOf("comment", "comments")),
|
||||||
|
}
|
||||||
|
|
||||||
|
class RirGeoFeedLoader(
|
||||||
|
private val source: String,
|
||||||
|
private val acceptedFields: Set<RirMetadataField>,
|
||||||
|
private val fetch: (String) -> String = RirGeoFeedLoader::httpGet,
|
||||||
|
) {
|
||||||
|
fun urls(registryObject: String): Set<String> =
|
||||||
|
registryObject
|
||||||
|
.lineSequence()
|
||||||
|
.map(String::trim)
|
||||||
|
.filter { ':' in it }
|
||||||
|
.filter { line ->
|
||||||
|
acceptedFields.any { field ->
|
||||||
|
line.substringBefore(':').trim().lowercase() in field.keys
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.flatMap { urlPattern.findAll(it).map { match -> match.value } }
|
||||||
|
.toSet()
|
||||||
|
|
||||||
|
fun pull(registryObject: String): List<GeoFeed> =
|
||||||
|
urls(registryObject).map { url -> GeoFeedCodec.decode(fetch(url), GeoFeedFormat.CSV) }
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
val urlPattern = Regex("https?://[^\\s,;\"]+")
|
||||||
|
val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
|
||||||
|
|
||||||
|
fun httpGet(url: String): String =
|
||||||
|
client
|
||||||
|
.send(
|
||||||
|
HttpRequest.newBuilder(URI(url)).GET().build(),
|
||||||
|
HttpResponse.BodyHandlers.ofString(),
|
||||||
|
)
|
||||||
|
.body()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
import java.net.URI
|
||||||
|
import java.net.http.HttpClient
|
||||||
|
import java.net.http.HttpRequest
|
||||||
|
import java.net.http.HttpResponse
|
||||||
|
import rip.crit.commonip.rawr.feeds.DelegatedStats
|
||||||
|
import rip.crit.commonip.rawr.types.Ingest
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
|
||||||
|
abstract class RirProvider(
|
||||||
|
private val name: String,
|
||||||
|
private val endpoint: String,
|
||||||
|
private val loader: (() -> String)? = null,
|
||||||
|
) : Ingest {
|
||||||
|
override fun ingest(): GeoFeed =
|
||||||
|
DelegatedStats.parse(
|
||||||
|
loader?.invoke()
|
||||||
|
?: client
|
||||||
|
.send(
|
||||||
|
HttpRequest.newBuilder(URI(endpoint)).GET().build(),
|
||||||
|
HttpResponse.BodyHandlers.ofString(),
|
||||||
|
)
|
||||||
|
.body(),
|
||||||
|
name,
|
||||||
|
)
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
package rip.crit.commonip.rawr.types
|
||||||
|
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
interface Ingest {
|
||||||
|
fun ingest(): GeoFeed
|
||||||
|
|
||||||
|
fun lookupOne(address: String): GeoLocation? =
|
||||||
|
ingest()
|
||||||
|
.records
|
||||||
|
.filter { Cidr.contains(it.prefix, address) }
|
||||||
|
.maxByOrNull { Cidr.prefixLength(it.prefix) }
|
||||||
|
}
|
||||||
|
|
||||||
|
object Cidr {
|
||||||
|
fun prefixLength(cidr: String): Int = cidr.substringAfter('/').toIntOrNull() ?: -1
|
||||||
|
|
||||||
|
fun contains(cidr: String, address: String): Boolean {
|
||||||
|
val parts = cidr.split('/', limit = 2)
|
||||||
|
val bits = parts.getOrNull(1)?.toIntOrNull() ?: return false
|
||||||
|
if (bits !in 0..32) return false
|
||||||
|
val network = ipv4(parts[0]) ?: return false
|
||||||
|
val target = ipv4(address) ?: return false
|
||||||
|
val mask = if (bits == 0) 0 else (-1 shl (32 - bits))
|
||||||
|
return network and mask == target and mask
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun ipv4(value: String): Int? {
|
||||||
|
val fields = value.split('.')
|
||||||
|
if (fields.size != 4) return null
|
||||||
|
return fields.fold(0) { result, field ->
|
||||||
|
field.toIntOrNull()?.takeIf { it in 0..255 }?.let { (result shl 8) or it } ?: return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
package rip.crit.commonip.rawr.providers
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class RirProviderTest {
|
||||||
|
@Test
|
||||||
|
fun `arin parses delegated ipv4 fixture`() {
|
||||||
|
val feed = ARINIngest { "arin|US|ipv4|192.0.2.0|256|20260101|allocated" }.ingest()
|
||||||
|
assertEquals("192.0.2.0/24", feed.records.single().prefix)
|
||||||
|
assertEquals("US", feed.records.single().countryCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
9
packages/serialize/build.gradle.kts
Normal file
9
packages/serialize/build.gradle.kts
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
/*
|
||||||
|
* This file was generated by the Gradle 'init' task.
|
||||||
|
*
|
||||||
|
* This project uses @Incubating APIs which are subject to change.
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-library-conventions")
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
package rip.crit.commonip.serialize
|
||||||
|
|
||||||
|
fun String.deserializeGeoFeed(format: GeoFeedFormat): GeoFeed = GeoFeedCodec.decode(this, format)
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
package rip.crit.commonip.serialize
|
||||||
|
|
||||||
|
import java.time.Instant
|
||||||
|
|
||||||
|
data class GeoLocation(
|
||||||
|
val prefix: String,
|
||||||
|
val countryCode: String,
|
||||||
|
val region: String? = null,
|
||||||
|
val city: String? = null,
|
||||||
|
val postalCode: String? = null,
|
||||||
|
val latitude: Double? = null,
|
||||||
|
val longitude: Double? = null,
|
||||||
|
val source: String = "unknown",
|
||||||
|
val observedAt: Instant = Instant.now(),
|
||||||
|
val confidence: Double = 0.5,
|
||||||
|
) {
|
||||||
|
init {
|
||||||
|
require('/' in prefix) { "prefix must be CIDR notation" }
|
||||||
|
require(countryCode.length == 2) { "countryCode must be ISO 3166-1 alpha-2" }
|
||||||
|
require(confidence in 0.0..1.0) { "confidence must be between 0 and 1" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data class GeoFeed(
|
||||||
|
val records: List<GeoLocation>,
|
||||||
|
val generatedAt: Instant = Instant.now(),
|
||||||
|
val schemaVersion: String = "commonip/geofeed/v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
enum class GeoFeedFormat {
|
||||||
|
CSV,
|
||||||
|
JSON,
|
||||||
|
XML,
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,158 @@
|
||||||
|
package rip.crit.commonip.serialize
|
||||||
|
|
||||||
|
import java.time.Instant
|
||||||
|
|
||||||
|
object GeoFeedCodec {
|
||||||
|
fun encode(feed: GeoFeed, format: GeoFeedFormat): String =
|
||||||
|
when (format) {
|
||||||
|
GeoFeedFormat.CSV ->
|
||||||
|
feed.records.joinToString("\n") { record ->
|
||||||
|
listOf(
|
||||||
|
record.prefix,
|
||||||
|
record.countryCode,
|
||||||
|
record.region.orEmpty(),
|
||||||
|
record.city.orEmpty(),
|
||||||
|
record.postalCode.orEmpty(),
|
||||||
|
)
|
||||||
|
.joinToString(",") { csv(it) }
|
||||||
|
}
|
||||||
|
GeoFeedFormat.JSON ->
|
||||||
|
"{\"schema\":\"${feed.schemaVersion}\",\"generatedAt\":\"${feed.generatedAt}\",\"records\":[" +
|
||||||
|
feed.records.joinToString(",") { jsonRecord(it) } +
|
||||||
|
"]}"
|
||||||
|
GeoFeedFormat.XML ->
|
||||||
|
"<geofeed schema=\"${xml(feed.schemaVersion)}\" generatedAt=\"${feed.generatedAt}\">" +
|
||||||
|
feed.records.joinToString("") { record ->
|
||||||
|
"<location prefix=\"${xml(record.prefix)}\" country=\"${xml(record.countryCode)}\"${attr("region", record.region)}${attr("city", record.city)}${attr("postalCode", record.postalCode)}${attr("latitude", record.latitude)}${attr("longitude", record.longitude)} source=\"${xml(record.source)}\" observedAt=\"${record.observedAt}\" confidence=\"${record.confidence}\"/>"
|
||||||
|
} +
|
||||||
|
"</geofeed>"
|
||||||
|
}
|
||||||
|
|
||||||
|
fun decode(payload: String, format: GeoFeedFormat): GeoFeed =
|
||||||
|
when (format) {
|
||||||
|
GeoFeedFormat.CSV ->
|
||||||
|
GeoFeed(
|
||||||
|
payload
|
||||||
|
.lineSequence()
|
||||||
|
.filter { it.isNotBlank() && !it.trimStart().startsWith("#") }
|
||||||
|
.map { csvRecord(it) }
|
||||||
|
.toList()
|
||||||
|
)
|
||||||
|
GeoFeedFormat.JSON ->
|
||||||
|
GeoFeed(
|
||||||
|
jsonObjects(
|
||||||
|
payload.substringAfter("\"records\":[", "").substringBeforeLast("]", "")
|
||||||
|
)
|
||||||
|
.map { jsonToRecord(it) }
|
||||||
|
)
|
||||||
|
GeoFeedFormat.XML ->
|
||||||
|
GeoFeed(
|
||||||
|
Regex("<location\\s+([^>]+)/>")
|
||||||
|
.findAll(payload)
|
||||||
|
.map { xmlToRecord(it.groupValues[1]) }
|
||||||
|
.toList()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun csvRecord(line: String): GeoLocation {
|
||||||
|
val fields = splitCsv(line)
|
||||||
|
require(fields.size >= 2) { "RFC 8805 record needs a prefix and country" }
|
||||||
|
return GeoLocation(
|
||||||
|
fields[0],
|
||||||
|
fields[1].uppercase(),
|
||||||
|
fields.getOrNull(2).blankToNull(),
|
||||||
|
fields.getOrNull(3).blankToNull(),
|
||||||
|
fields.getOrNull(4).blankToNull(),
|
||||||
|
source = "rfc8805",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun jsonRecord(r: GeoLocation) =
|
||||||
|
"{" +
|
||||||
|
listOf(
|
||||||
|
"prefix" to r.prefix,
|
||||||
|
"countryCode" to r.countryCode,
|
||||||
|
"region" to r.region,
|
||||||
|
"city" to r.city,
|
||||||
|
"postalCode" to r.postalCode,
|
||||||
|
"latitude" to r.latitude,
|
||||||
|
"longitude" to r.longitude,
|
||||||
|
"source" to r.source,
|
||||||
|
"observedAt" to r.observedAt.toString(),
|
||||||
|
"confidence" to r.confidence,
|
||||||
|
)
|
||||||
|
.joinToString(",") { (k, v) ->
|
||||||
|
"\"$k\":" +
|
||||||
|
when (v) {
|
||||||
|
null -> "null"
|
||||||
|
is Number -> v.toString()
|
||||||
|
else -> "\"${json(v.toString())}\""
|
||||||
|
}
|
||||||
|
} +
|
||||||
|
"}"
|
||||||
|
|
||||||
|
private fun jsonToRecord(obj: String): GeoLocation {
|
||||||
|
fun value(name: String) =
|
||||||
|
Regex("\"$name\"\\s*:\\s*(null|\"((?:\\\\.|[^\"])*)\"|[-.0-9]+)").find(obj)?.let {
|
||||||
|
if (it.groupValues[1] == "null") null
|
||||||
|
else
|
||||||
|
it.groupValues[2]
|
||||||
|
.ifEmpty { it.groupValues[1] }
|
||||||
|
.replace("\\\"", "\"")
|
||||||
|
.replace("\\\\", "\\")
|
||||||
|
}
|
||||||
|
return GeoLocation(
|
||||||
|
value("prefix") ?: error("JSON record missing prefix"),
|
||||||
|
value("countryCode") ?: error("JSON record missing countryCode"),
|
||||||
|
value("region"),
|
||||||
|
value("city"),
|
||||||
|
value("postalCode"),
|
||||||
|
value("latitude")?.toDouble(),
|
||||||
|
value("longitude")?.toDouble(),
|
||||||
|
value("source") ?: "commonip",
|
||||||
|
value("observedAt")?.let(Instant::parse) ?: Instant.now(),
|
||||||
|
value("confidence")?.toDouble() ?: 0.5,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun xmlToRecord(attrs: String): GeoLocation {
|
||||||
|
fun value(name: String) =
|
||||||
|
Regex("$name=\"([^\"]*)\"").find(attrs)?.groupValues?.get(1)?.let {
|
||||||
|
it.replace(""", "\"").replace("&", "&")
|
||||||
|
}
|
||||||
|
return GeoLocation(
|
||||||
|
value("prefix") ?: error("XML record missing prefix"),
|
||||||
|
value("country") ?: error("XML record missing country"),
|
||||||
|
value("region"),
|
||||||
|
value("city"),
|
||||||
|
value("postalCode"),
|
||||||
|
value("latitude")?.toDouble(),
|
||||||
|
value("longitude")?.toDouble(),
|
||||||
|
value("source") ?: "commonip",
|
||||||
|
value("observedAt")?.let(Instant::parse) ?: Instant.now(),
|
||||||
|
value("confidence")?.toDouble() ?: 0.5,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun jsonObjects(raw: String): List<String> =
|
||||||
|
Regex("\\{(?:[^{}]|\\{[^{}]*})*}").findAll(raw).map { it.value }.toList()
|
||||||
|
|
||||||
|
private fun splitCsv(line: String): List<String> =
|
||||||
|
Regex("(?:^|,)(\"(?:[^\"]|\"\")*\"|[^,]*)")
|
||||||
|
.findAll(line)
|
||||||
|
.map { it.groupValues[1].removeSurrounding("\"").replace("\"\"", "\"") }
|
||||||
|
.toList()
|
||||||
|
|
||||||
|
private fun csv(value: String) =
|
||||||
|
if (value.any { it == ',' || it == '"' || it == '\n' }) "\"${value.replace("\"", "\"\"")}\""
|
||||||
|
else value
|
||||||
|
|
||||||
|
private fun json(value: String) = value.replace("\\", "\\\\").replace("\"", "\\\"")
|
||||||
|
|
||||||
|
private fun xml(value: String) = value.replace("&", "&").replace("\"", """)
|
||||||
|
|
||||||
|
private fun attr(name: String, value: Any?) =
|
||||||
|
value?.let { " $name=\"${xml(it.toString())}\"" }.orEmpty()
|
||||||
|
|
||||||
|
private fun String?.blankToNull() = this?.takeIf { it.isNotBlank() }
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
package rip.crit.commonip.serialize
|
||||||
|
|
||||||
|
fun GeoFeed.serialize(format: GeoFeedFormat): String = GeoFeedCodec.encode(this, format)
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
package rip.crit.commonip.serialize
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class GeoFeedCodecTest {
|
||||||
|
@Test
|
||||||
|
fun `RFC 8805 records round trip`() {
|
||||||
|
val feed = GeoFeed(listOf(GeoLocation("192.0.2.0/24", "US", "PA", "Philadelphia", "19106")))
|
||||||
|
val parsed =
|
||||||
|
GeoFeedCodec.decode(GeoFeedCodec.encode(feed, GeoFeedFormat.CSV), GeoFeedFormat.CSV)
|
||||||
|
assertEquals(feed.records.first().prefix, parsed.records.first().prefix)
|
||||||
|
assertEquals("Philadelphia", parsed.records.first().city)
|
||||||
|
}
|
||||||
|
}
|
||||||
3
packages/settings.gradle.kts
Normal file
3
packages/settings.gradle.kts
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
rootProject.name = "commonip-packages"
|
||||||
|
include("decentraland", "serialize", "ml", "rawr")
|
||||||
74
proprietary/build.gradle.kts
Normal file
74
proprietary/build.gradle.kts
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
import java.net.URI
|
||||||
|
import javax.xml.parsers.DocumentBuilderFactory
|
||||||
|
import org.w3c.dom.Element
|
||||||
|
import org.w3c.dom.Node
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("buildlogic.kotlin-library-conventions")
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
api(project(":packages:serialize"))
|
||||||
|
implementation(project(":packages:rawr"))
|
||||||
|
implementation("commons-net:commons-net:3.11.1")
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.register("whois") {
|
||||||
|
group = "generation"
|
||||||
|
description = "Fetches latest WHOIS server mapping and generates WhoisServers.kt in source."
|
||||||
|
|
||||||
|
val outputDirectory = file("src/main/kotlin/rip/crit/commonip/proprietary")
|
||||||
|
outputs.file(outputDirectory.resolve("WhoisServers.kt"))
|
||||||
|
|
||||||
|
doLast {
|
||||||
|
val xmlUrl =
|
||||||
|
"https://raw.githubusercontent.com/whois-server-list/whois-server-list/refs/heads/master/whois-server-list.xml"
|
||||||
|
|
||||||
|
val inputStream = URI(xmlUrl).toURL().openStream()
|
||||||
|
val dbFactory = DocumentBuilderFactory.newInstance()
|
||||||
|
val dBuilder = dbFactory.newDocumentBuilder()
|
||||||
|
val xmlDoc = dBuilder.parse(inputStream)
|
||||||
|
xmlDoc.documentElement.normalize()
|
||||||
|
|
||||||
|
val serverMap = mutableMapOf<String, String>()
|
||||||
|
val domainNodes = xmlDoc.getElementsByTagName("domain")
|
||||||
|
|
||||||
|
for (i in 0 until domainNodes.length) {
|
||||||
|
val domainNode = domainNodes.item(i)
|
||||||
|
if (domainNode.nodeType == Node.ELEMENT_NODE) {
|
||||||
|
val domainElement = domainNode as Element
|
||||||
|
val domainName = domainElement.getAttribute("name")
|
||||||
|
|
||||||
|
val whoisServerNodes = domainElement.getElementsByTagName("whoisServer")
|
||||||
|
if (whoisServerNodes.length > 0) {
|
||||||
|
val whoisServerElement = whoisServerNodes.item(0) as Element
|
||||||
|
val host = whoisServerElement.getAttribute("host")
|
||||||
|
if (host.isNotBlank() && domainName.isNotBlank()) {
|
||||||
|
serverMap[domainName] = host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val mapEntries = StringBuilder()
|
||||||
|
serverMap.forEach { (tld, server) ->
|
||||||
|
mapEntries.append(" \"$tld\" to \"$server\",\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
val outputFile = outputDirectory.resolve("WhoisServers.kt")
|
||||||
|
outputFile.parentFile.mkdirs()
|
||||||
|
outputFile.writeText(
|
||||||
|
"""
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
// Auto-generated file. Do not edit directly.
|
||||||
|
object WhoisServers {
|
||||||
|
val map: Map<String, String> = mapOf(
|
||||||
|
$mapEntries )
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
.trimIndent()
|
||||||
|
)
|
||||||
|
println("Successfully generated ${outputFile.path} with ${serverMap.size} active TLD servers.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import java.net.URI
|
||||||
|
import java.net.http.HttpClient
|
||||||
|
import java.net.http.HttpRequest
|
||||||
|
import java.net.http.HttpResponse
|
||||||
|
|
||||||
|
fun interface HttpTransport {
|
||||||
|
fun get(url: String, headers: Map<String, String>): String
|
||||||
|
}
|
||||||
|
|
||||||
|
object DefaultHttpTransport : HttpTransport {
|
||||||
|
private val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
|
||||||
|
|
||||||
|
override fun get(url: String, headers: Map<String, String>): String {
|
||||||
|
val request = HttpRequest.newBuilder(URI(url)).apply { headers.forEach(::header) }.GET().build()
|
||||||
|
return client.send(request, HttpResponse.BodyHandlers.ofString()).body()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import rip.crit.commonip.rawr.types.Cidr
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||||
|
import rip.crit.commonip.serialize.GeoLocation
|
||||||
|
|
||||||
|
class OpenGeoFeed(private val transport: HttpTransport = DefaultHttpTransport) {
|
||||||
|
fun pull(url: String): GeoFeed =
|
||||||
|
GeoFeedCodec.decode(transport.get(url, emptyMap()), GeoFeedFormat.CSV)
|
||||||
|
|
||||||
|
fun lookup(url: String, address: String): GeoLocation? =
|
||||||
|
pull(url)
|
||||||
|
.records
|
||||||
|
.filter { Cidr.contains(it.prefix, address) }
|
||||||
|
.maxByOrNull { Cidr.prefixLength(it.prefix) }
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
class Rdap(private val transport: HttpTransport = DefaultHttpTransport) {
|
||||||
|
fun lookup(address: String): String =
|
||||||
|
transport.get("https://rdap.org/ip/$address", mapOf("Accept" to "application/rdap+json"))
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import java.time.Duration
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.Semaphore
|
||||||
|
|
||||||
|
class RequestGovernor(
|
||||||
|
private val minimumInterval: Duration = Duration.ofMillis(500),
|
||||||
|
parallelism: Int = 2,
|
||||||
|
) {
|
||||||
|
private val permits = Semaphore(parallelism)
|
||||||
|
private val lastRequest = ConcurrentHashMap<String, Long>()
|
||||||
|
|
||||||
|
fun <T> execute(provider: String, action: () -> T): T {
|
||||||
|
permits.acquire()
|
||||||
|
try {
|
||||||
|
synchronized(lastRequest) {
|
||||||
|
val wait =
|
||||||
|
minimumInterval.toMillis() - (System.currentTimeMillis() - (lastRequest[provider] ?: 0))
|
||||||
|
if (wait > 0) Thread.sleep(wait)
|
||||||
|
lastRequest[provider] = System.currentTimeMillis()
|
||||||
|
}
|
||||||
|
return action()
|
||||||
|
} finally {
|
||||||
|
permits.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun <T> chunks(values: Collection<T>, size: Int = 100): Sequence<List<T>> =
|
||||||
|
values.asSequence().chunked(size)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import rip.crit.commonip.serialize.GeoFeed
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||||
|
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||||
|
|
||||||
|
class StarlinkGeoFeed(private val transport: HttpTransport = DefaultHttpTransport) {
|
||||||
|
companion object {
|
||||||
|
const val URL = "https://geoip.starlinkisp.net/feed.csv"
|
||||||
|
}
|
||||||
|
|
||||||
|
fun pull(): GeoFeed = GeoFeedCodec.decode(transport.get(URL, emptyMap()), GeoFeedFormat.CSV)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import org.apache.commons.net.whois.WhoisClient
|
||||||
|
|
||||||
|
class Whois(private val servers: Map<String, String> = WhoisServers.map) {
|
||||||
|
fun lookup(query: String): String {
|
||||||
|
val server =
|
||||||
|
servers[query.substringAfterLast('.', "").lowercase()] ?: servers.getValue("default")
|
||||||
|
val client = WhoisClient()
|
||||||
|
return try {
|
||||||
|
client.connect(server)
|
||||||
|
client.query("$query\r\n")
|
||||||
|
} finally {
|
||||||
|
if (client.isConnected) client.disconnect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,13 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable
|
||||||
|
|
||||||
|
class LiveProviderTest {
|
||||||
|
@Test
|
||||||
|
@EnabledIfEnvironmentVariable(named = "COMMONIP_LIVE_TESTS", matches = "true")
|
||||||
|
fun `starlink live feed has records in isolated opt in test`() {
|
||||||
|
assertTrue(StarlinkGeoFeed().pull().records.isNotEmpty())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
package rip.crit.commonip.proprietary
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class ProviderTest {
|
||||||
|
@Test
|
||||||
|
fun `open geofeed uses injected fixture transport`() {
|
||||||
|
val feed =
|
||||||
|
OpenGeoFeed(HttpTransport { _, _ -> "203.0.113.0/24,US,CA,Los Angeles,90001" })
|
||||||
|
.pull("https://fixture.invalid/feed.csv")
|
||||||
|
assertEquals("Los Angeles", feed.records.single().city)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `starlink endpoint is opt in provider`() {
|
||||||
|
assertEquals("https://geoip.starlinkisp.net/feed.csv", StarlinkGeoFeed.URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
12
scripts/commonip.service
Normal file
12
scripts/commonip.service
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
[Unit]
|
||||||
|
Description=CommonIP node
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=commonip
|
||||||
|
WorkingDirectory=/opt/commonip
|
||||||
|
ExecStart=/opt/commonip/commonip
|
||||||
|
Restart=on-failure
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
14
scripts/deploy-ssh.sh
Executable file
14
scripts/deploy-ssh.sh
Executable file
|
|
@ -0,0 +1,14 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
deploy_host="${1:?host is required}"
|
||||||
|
deploy_user="${2:?user is required}"
|
||||||
|
deploy_path="${COMMONIP_REMOTE_PATH:-/opt/commonip}"
|
||||||
|
|
||||||
|
./gradlew :app:nativeImage
|
||||||
|
|
||||||
|
ssh "${deploy_user}@${deploy_host}" "sudo useradd --system --create-home --shell /usr/sbin/nologin commonip 2>/dev/null || true; sudo install -d -o commonip -g commonip ${deploy_path}"
|
||||||
|
scp app/build/native/commonip "${deploy_user}@${deploy_host}:/tmp/commonip"
|
||||||
|
ssh "${deploy_user}@${deploy_host}" "sudo install -o commonip -g commonip -m 0755 /tmp/commonip ${deploy_path}/commonip"
|
||||||
|
scp scripts/commonip.service "${deploy_user}@${deploy_host}:/tmp/commonip.service"
|
||||||
|
ssh "${deploy_user}@${deploy_host}" "sudo install -m 0644 /tmp/commonip.service /etc/systemd/system/commonip.service; sudo systemctl daemon-reload; sudo systemctl enable --now commonip"
|
||||||
13
settings.gradle.kts
Normal file
13
settings.gradle.kts
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
pluginManagement {
|
||||||
|
includeBuild("build-logic")
|
||||||
|
}
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0"
|
||||||
|
}
|
||||||
|
|
||||||
|
rootProject.name = "commonip"
|
||||||
|
|
||||||
|
include("app", "proprietary", "packages")
|
||||||
|
include(":packages:serialize", ":packages:rawr", ":packages:ml", ":packages:decentraland")
|
||||||
|
include(":packages:decentraland:kademlia", ":packages:decentraland:git", ":packages:decentraland:chksum")
|
||||||
Loading…
Reference in a new issue