Initial commit.
This commit is contained in:
commit
89a7c8eaef
90 changed files with 3920 additions and 0 deletions
12
.gitattributes
vendored
Normal file
12
.gitattributes
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
#
|
||||
# https://help.github.com/articles/dealing-with-line-endings/
|
||||
#
|
||||
# Linux start script should use lf
|
||||
/gradlew text eol=lf
|
||||
|
||||
# These are Windows script files and should use crlf
|
||||
*.bat text eol=crlf
|
||||
|
||||
# Binary files should be left untouched
|
||||
*.jar binary
|
||||
|
||||
12
.gitignore
vendored
Normal file
12
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Ignore Gradle project-specific cache directory
|
||||
.gradle
|
||||
|
||||
# Ignore Gradle build output directory
|
||||
build
|
||||
|
||||
# Ignore Kotlin plugin data
|
||||
.kotlin
|
||||
|
||||
# Node credentials and deployment-specific endpoints
|
||||
node.yaml
|
||||
deploy/*.yaml
|
||||
16
Justfile
Normal file
16
Justfile
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
|
||||
|
||||
build:
|
||||
./gradlew build
|
||||
|
||||
fmt:
|
||||
./gradlew spotlessApply
|
||||
|
||||
check:
|
||||
./gradlew spotlessCheck build
|
||||
|
||||
native:
|
||||
./gradlew nativeImage
|
||||
|
||||
deploy host user:
|
||||
scripts/deploy-ssh.sh {{host}} {{user}}
|
||||
12
README.md
Normal file
12
README.md
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# CommonIP
|
||||
|
||||
CommonIP, not to be confused with [CommonApp](https://commonapp.org), is a decentralized geolocation network that specifically avoids proprietary geolocation services whenever possible. It's written in the ever-so-portable Kotlin. Use it from here, use it from there, use it from anywhere!
|
||||
|
||||
# donate
|
||||
|
||||
Please donate to me to keep this project and its (centralized) [geofeeds](https://ai.crit.rip/automatedintelligence/geofeeds) alive.
|
||||
|
||||
- `eth` 0xF30C281Aa2CD94618D902AE5bb9101E9F5385fE3
|
||||
- `sol` 99zJPASUz9PKLFaJWrxHCEtLZxeEePYpgZ74Ly9YNUHy
|
||||
- `paypal` +1 (267) 806-2744 - Stephen Hellings
|
||||
- `zelle` +1 (267) 806-2744 - STEPHEN HELLINGS
|
||||
32
app/build.gradle.kts
Normal file
32
app/build.gradle.kts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import buildlogic.GraalNativeImageTask
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-application-conventions")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("org.apache.commons:commons-text")
|
||||
implementation(project(":packages:serialize"))
|
||||
implementation(project(":packages:rawr"))
|
||||
implementation(project(":proprietary"))
|
||||
implementation(project(":packages:ml"))
|
||||
implementation(project(":packages:decentraland"))
|
||||
}
|
||||
|
||||
application {
|
||||
mainClass = "rip.crit.commonip.app.MainKtKt"
|
||||
}
|
||||
|
||||
tasks.named<JavaExec>("run") {
|
||||
standardInput = System.`in`
|
||||
}
|
||||
|
||||
val nativeImage =
|
||||
tasks.register<GraalNativeImageTask>("nativeImage") {
|
||||
group = "distribution"
|
||||
description = "Builds a GraalVM native executable with the active Gradle toolchain."
|
||||
dependsOn(tasks.named("classes"))
|
||||
classpath.set(sourceSets.main.get().runtimeClasspath.files.map { it.absolutePath })
|
||||
mainClass.set("rip.crit.commonip.app.MainKtKt")
|
||||
output.set(layout.buildDirectory.file("native/commonip"))
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
package rip.crit.commonip.app
|
||||
|
||||
import java.net.InetSocketAddress
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import rip.crit.commonip.decentraland.git.RepositoryTrust
|
||||
import rip.crit.commonip.decentraland.kademlia.KademliaNode
|
||||
import rip.crit.commonip.decentraland.kademlia.NodeRegistry
|
||||
import rip.crit.commonip.ml.GeoLocation as MlGeoLocation
|
||||
import rip.crit.commonip.ml.LocationEstimator
|
||||
import rip.crit.commonip.proprietary.OpenGeoFeed
|
||||
import rip.crit.commonip.proprietary.StarlinkGeoFeed
|
||||
import rip.crit.commonip.rawr.providers.RirGeoFeedLoader
|
||||
import rip.crit.commonip.rawr.providers.RirMetadataField
|
||||
import rip.crit.commonip.rawr.types.Cidr
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
class ClusterGeolocation(private val registry: NodeRegistry, bind: InetSocketAddress) :
|
||||
AutoCloseable {
|
||||
private val feeds = ConcurrentHashMap<String, RepositoryTrust>()
|
||||
private val local = ConcurrentHashMap<String, GeoLocation>()
|
||||
private val dht = KademliaNode(bind, registry)
|
||||
|
||||
fun start(seeds: List<InetSocketAddress> = emptyList()) {
|
||||
dht.start()
|
||||
seeds.forEach(dht::bootstrap)
|
||||
}
|
||||
|
||||
fun addFeed(url: String, trust: RepositoryTrust = RepositoryTrust.COMMUNITY) {
|
||||
feeds[url] = trust
|
||||
}
|
||||
|
||||
fun addStarlinkFeed() {
|
||||
feeds[StarlinkGeoFeed.URL] = RepositoryTrust.TAINTED
|
||||
}
|
||||
|
||||
fun pullRirRegistryObject(
|
||||
source: String,
|
||||
registryObject: String,
|
||||
fields: Set<RirMetadataField>,
|
||||
trust: RepositoryTrust = RepositoryTrust.COMMUNITY,
|
||||
) {
|
||||
RirGeoFeedLoader(source, fields).pull(registryObject).forEach { feed ->
|
||||
feed.records.forEach { record ->
|
||||
local[record.prefix] = record
|
||||
dht.store(
|
||||
"${if (trust == RepositoryTrust.TAINTED) "commonip/tainted-full" else "commonip/community"}:${record.prefix}",
|
||||
record.prefix,
|
||||
)
|
||||
}
|
||||
registry.publish(
|
||||
"local",
|
||||
"rir geofeed pulled",
|
||||
"$source yielded ${feed.records.size} records",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun pullAll() {
|
||||
feeds.forEach { (url, trust) ->
|
||||
runCatching { OpenGeoFeed().pull(url).records }
|
||||
.onSuccess { records ->
|
||||
records.forEach { record ->
|
||||
local[record.prefix] = record
|
||||
dht.store(
|
||||
"${if (trust == RepositoryTrust.TAINTED) "commonip/tainted-full" else "commonip/community"}:${record.prefix}",
|
||||
record.prefix,
|
||||
)
|
||||
}
|
||||
registry.publish(
|
||||
"local",
|
||||
"${trust.name.lowercase()} feed pulled",
|
||||
"$url yielded ${records.size} records",
|
||||
)
|
||||
}
|
||||
.onFailure { registry.publish("local", "feed pull failed", "$url ${it.message}") }
|
||||
}
|
||||
}
|
||||
|
||||
fun lookup(address: String): GeoLocation? =
|
||||
local.values
|
||||
.filter { Cidr.contains(it.prefix, address) }
|
||||
.maxByOrNull { Cidr.prefixLength(it.prefix) }
|
||||
|
||||
fun estimate(address: String) =
|
||||
LocationEstimator()
|
||||
.estimate(
|
||||
local.values
|
||||
.filter { Cidr.contains(it.prefix, address) }
|
||||
.map { MlGeoLocation(it.latitude, it.longitude, it.confidence) }
|
||||
)
|
||||
|
||||
override fun close() = dht.close()
|
||||
}
|
||||
116
app/src/main/kotlin/rip/crit/commonip/app/MainKt.kt
Normal file
116
app/src/main/kotlin/rip/crit/commonip/app/MainKt.kt
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
package rip.crit.commonip.app
|
||||
|
||||
import java.net.InetSocketAddress
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Files
|
||||
import rip.crit.commonip.decentraland.git.NodeYamlReader
|
||||
import rip.crit.commonip.decentraland.SshDashboardService
|
||||
import rip.crit.commonip.decentraland.kademlia.NetworkNode
|
||||
import rip.crit.commonip.decentraland.kademlia.NodeRegistry
|
||||
import rip.crit.commonip.decentraland.kademlia.NodeState
|
||||
|
||||
fun main() = CommonIpNode().run()
|
||||
|
||||
private class CommonIpNode {
|
||||
private val registry = NodeRegistry()
|
||||
private val cluster =
|
||||
ClusterGeolocation(
|
||||
registry,
|
||||
InetSocketAddress(System.getenv("COMMONIP_DHT_PORT")?.toIntOrNull() ?: 7400),
|
||||
)
|
||||
|
||||
fun run() {
|
||||
val configPath = Path.of(System.getenv("COMMONIP_NODE_CONFIG") ?: "/etc/commonip/node.yaml")
|
||||
val configuredBootstrap = if (Files.isRegularFile(configPath)) NodeYamlReader.parse(Files.readString(configPath)).bootstrap else emptyList()
|
||||
cluster.start((configuredBootstrap + System.getenv("COMMONIP_BOOTSTRAP")?.split(',').orEmpty()).mapNotNull(::parseEndpoint).distinct())
|
||||
System.getenv("COMMONIP_GEOFEEDS")
|
||||
?.split(',')
|
||||
?.filter(String::isNotBlank)
|
||||
?.forEach(cluster::addFeed)
|
||||
if (System.getenv("COMMONIP_ENABLE_TAINTED_STARLINK") == "true") cluster.addStarlinkFeed()
|
||||
cluster.pullAll()
|
||||
val sshService =
|
||||
System.getenv("COMMONIP_SSH_PASSWD")?.let { password ->
|
||||
SshDashboardService(
|
||||
System.getenv("COMMONIP_SSH_PORT")?.toIntOrNull() ?: 2222,
|
||||
Path.of(System.getenv("COMMONIP_SSH_HOST_KEY") ?: "/opt/commonip/hostkey.ser"),
|
||||
System.getenv("COMMONIP_SSH_USER") ?: "commonip",
|
||||
password.toCharArray(),
|
||||
) {
|
||||
registry.snapshot().joinToString("\n") { "${it.id} ${it.state}" }
|
||||
}
|
||||
.also { it.start() }
|
||||
}
|
||||
if (System.console() == null) {
|
||||
while (true) {
|
||||
Thread.sleep(60_000)
|
||||
cluster.pullAll()
|
||||
}
|
||||
}
|
||||
sshService?.close()
|
||||
while (true) {
|
||||
render()
|
||||
print("commonip/network> ")
|
||||
val command = readlnOrNull()?.trim()?.lowercase() ?: return
|
||||
when (command) {
|
||||
"q",
|
||||
"quit",
|
||||
"exit" -> return
|
||||
"r",
|
||||
"refresh" -> cluster.pullAll()
|
||||
"l",
|
||||
"lookup" -> lookup()
|
||||
"e",
|
||||
"estimate" -> estimate()
|
||||
else -> println("Commands are r refresh, l lookup, e estimate, q quit.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseEndpoint(value: String): InetSocketAddress? =
|
||||
value.trim().split(':', limit = 2).let { parts ->
|
||||
parts.getOrNull(1)?.toIntOrNull()?.let { InetSocketAddress(parts[0], it) }
|
||||
}
|
||||
|
||||
private fun render() {
|
||||
print("\u001b[H\u001b[2J")
|
||||
val nodes = registry.snapshot()
|
||||
println()
|
||||
nodes.chunked(8).forEach { row ->
|
||||
println(row.joinToString(" ") { node -> "${status(node)} ${node.id}" })
|
||||
}
|
||||
println()
|
||||
registry.feed().forEach { event ->
|
||||
println(
|
||||
"${event.at.toString().substring(11, 19)} ${event.nodeId} ${event.action} ${event.detail}"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun status(node: NetworkNode): String =
|
||||
when (node.state) {
|
||||
NodeState.ONLINE -> "\u001b[32m●\u001b[0m"
|
||||
NodeState.DEGRADED -> "\u001b[33m●\u001b[0m"
|
||||
NodeState.OFFLINE -> "\u001b[90m●\u001b[0m"
|
||||
}
|
||||
|
||||
private fun lookup() {
|
||||
print("IPv4 address> ")
|
||||
val address = readlnOrNull()?.trim().orEmpty()
|
||||
val match = cluster.lookup(address)
|
||||
registry.publish("local", "lookup", match?.prefix ?: "no matching CIDR for $address")
|
||||
}
|
||||
|
||||
private fun estimate() {
|
||||
print("CIDR or IPv4 address> ")
|
||||
val address = readlnOrNull()?.trim().orEmpty()
|
||||
val result = cluster.estimate(address)
|
||||
registry.publish(
|
||||
"local",
|
||||
"estimate",
|
||||
result?.let {
|
||||
"%.4f, %.4f confidence %.2f".format(it.latitude, it.longitude, it.confidence)
|
||||
} ?: "no coordinate observations",
|
||||
)
|
||||
}
|
||||
}
|
||||
18
build-logic/build.gradle.kts
Normal file
18
build-logic/build.gradle.kts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
`kotlin-dsl`
|
||||
}
|
||||
|
||||
repositories {
|
||||
gradlePluginPortal()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation(libs.kotlin.gradle.plugin)
|
||||
implementation("com.diffplug.spotless:com.diffplug.spotless.gradle.plugin:8.10.2")
|
||||
}
|
||||
14
build-logic/settings.gradle.kts
Normal file
14
build-logic/settings.gradle.kts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This settings file is used to specify which projects to include in your build-logic build.
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
dependencyResolutionManagement {
|
||||
versionCatalogs {
|
||||
create("libs", { from(files("../gradle/libs.versions.toml")) })
|
||||
}
|
||||
}
|
||||
|
||||
rootProject.name = "build-logic"
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-common-conventions")
|
||||
|
||||
application
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("org.jetbrains.kotlin.jvm")
|
||||
id("com.diffplug.spotless")
|
||||
}
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
constraints {
|
||||
implementation("org.apache.commons:commons-text:1.14.0")
|
||||
}
|
||||
}
|
||||
|
||||
testing {
|
||||
suites {
|
||||
val test = named<JvmTestSuite>("test") {
|
||||
useJUnitJupiter("6.0.1")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.named("check") {
|
||||
dependsOn("spotlessCheck")
|
||||
}
|
||||
|
||||
java {
|
||||
toolchain {
|
||||
languageVersion = JavaLanguageVersion.of(25)
|
||||
vendor = JvmVendorSpec.GRAAL_VM
|
||||
}
|
||||
}
|
||||
|
||||
spotless {
|
||||
kotlin {
|
||||
target("src/**/*.kt")
|
||||
ktfmt()
|
||||
trimTrailingWhitespace()
|
||||
endWithNewline()
|
||||
}
|
||||
kotlinGradle {
|
||||
target("*.gradle.kts")
|
||||
ktfmt()
|
||||
trimTrailingWhitespace()
|
||||
endWithNewline()
|
||||
}
|
||||
}
|
||||
17
build-logic/src/main/kotlin/buildlogic.kotlin-fmt.gradle.kts
Normal file
17
build-logic/src/main/kotlin/buildlogic.kotlin-fmt.gradle.kts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
plugins {
|
||||
id("com.diffplug.spotless")
|
||||
}
|
||||
|
||||
spotless {
|
||||
java {
|
||||
target("src/**/*.java")
|
||||
importOrder()
|
||||
removeUnusedImports()
|
||||
palantirJavaFormat()
|
||||
}
|
||||
|
||||
kotlin {
|
||||
target("src/**/*.kt")
|
||||
ktlint()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-common-conventions")
|
||||
|
||||
`java-library`
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package buildlogic
|
||||
|
||||
import javax.inject.Inject
|
||||
import org.gradle.api.DefaultTask
|
||||
import org.gradle.api.file.RegularFileProperty
|
||||
import org.gradle.api.provider.ListProperty
|
||||
import org.gradle.api.tasks.Classpath
|
||||
import org.gradle.api.tasks.OutputFile
|
||||
import org.gradle.api.tasks.TaskAction
|
||||
import org.gradle.jvm.toolchain.JavaLanguageVersion
|
||||
import org.gradle.jvm.toolchain.JavaToolchainService
|
||||
import org.gradle.jvm.toolchain.JvmVendorSpec
|
||||
import org.gradle.process.ExecOperations
|
||||
|
||||
abstract class GraalNativeImageTask @Inject constructor(private val toolchains: JavaToolchainService, private val exec: ExecOperations) : DefaultTask() {
|
||||
@get:Classpath abstract val classpath: ListProperty<String>
|
||||
@get:OutputFile abstract val output: RegularFileProperty
|
||||
abstract val mainClass: org.gradle.api.provider.Property<String>
|
||||
|
||||
@TaskAction fun build() {
|
||||
val launcher = toolchains.launcherFor {
|
||||
languageVersion.set(JavaLanguageVersion.of(25))
|
||||
vendor.set(JvmVendorSpec.GRAAL_VM)
|
||||
}.get()
|
||||
val nativeImage = launcher.metadata.installationPath.file("bin/native-image").asFile
|
||||
check(nativeImage.canExecute()) { "The configured Java 25 toolchain must be GraalVM with native-image installed" }
|
||||
output.get().asFile.parentFile.mkdirs()
|
||||
exec.exec { commandLine(nativeImage.absolutePath, "-cp", classpath.get().joinToString(java.io.File.pathSeparator), mainClass.get(), output.get().asFile.absolutePath) }
|
||||
}
|
||||
}
|
||||
7
gradle.properties
Normal file
7
gradle.properties
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# This file was generated by the Gradle 'init' task.
|
||||
# https://docs.gradle.org/current/userguide/build_environment.html#sec:gradle_configuration_properties
|
||||
|
||||
org.gradle.configuration-cache=true
|
||||
org.gradle.parallel=true
|
||||
org.gradle.caching=true
|
||||
org.gradle.java.installations.auto-download=true
|
||||
8
gradle/libs.versions.toml
Normal file
8
gradle/libs.versions.toml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# This file was generated by the Gradle 'init' task.
|
||||
# https://docs.gradle.org/current/userguide/version_catalogs.html#sec::toml-dependencies-format
|
||||
|
||||
[versions]
|
||||
kotlin-gradle-plugin = "2.4.0"
|
||||
|
||||
[libraries]
|
||||
kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin-gradle-plugin" }
|
||||
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
Binary file not shown.
9
gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
9
gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.1-bin.zip
|
||||
networkTimeout=10000
|
||||
retries=0
|
||||
retryBackOffMs=500
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
248
gradlew
vendored
Executable file
248
gradlew
vendored
Executable file
|
|
@ -0,0 +1,248 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# gradlew start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh gradlew
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
82
gradlew.bat
vendored
Normal file
82
gradlew.bat
vendored
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
@rem SPDX-License-Identifier: Apache-2.0
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%"=="" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem gradlew startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables, and ensure extensions are enabled
|
||||
setlocal EnableExtensions
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%"=="" set DIRNAME=.
|
||||
@rem This is normally unused
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if %ERRORLEVEL% equ 0 goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
"%COMSPEC%" /c exit 1
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
"%COMSPEC%" /c exit 1
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
|
||||
|
||||
@rem Execute gradlew
|
||||
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
|
||||
@rem which allows us to clear the local environment before executing the java command
|
||||
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
|
||||
|
||||
:exitWithErrorLevel
|
||||
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
|
||||
"%COMSPEC%" /c exit %ERRORLEVEL%
|
||||
7
node.yaml.example
Normal file
7
node.yaml.example
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
remote: https://ai.crit.rip/automatedintelligence/geofeeds.git
|
||||
branch: main
|
||||
lfs: true
|
||||
tainted: false
|
||||
passwd: "set-through-secret-manager"
|
||||
bootstrap:
|
||||
- "peer.example.net:7400"
|
||||
1
packages/build.gradle.kts
Normal file
1
packages/build.gradle.kts
Normal file
|
|
@ -0,0 +1 @@
|
|||
plugins { base }
|
||||
10
packages/decentraland/build.gradle.kts
Normal file
10
packages/decentraland/build.gradle.kts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
plugins {
|
||||
id("buildlogic.kotlin-library-conventions")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api(project(":packages:serialize"))
|
||||
api(project(":packages:decentraland:kademlia"))
|
||||
api(project(":packages:decentraland:git"))
|
||||
implementation("org.apache.sshd:sshd-core:2.15.0")
|
||||
}
|
||||
1
packages/decentraland/chksum/build.gradle.kts
Normal file
1
packages/decentraland/chksum/build.gradle.kts
Normal file
|
|
@ -0,0 +1 @@
|
|||
plugins { id("buildlogic.kotlin-library-conventions") }
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package rip.crit.commonip.decentraland.chksum
|
||||
|
||||
import java.security.MessageDigest
|
||||
|
||||
object ContentHash {
|
||||
fun sha256(value: String): String =
|
||||
MessageDigest.getInstance("SHA-256").digest(value.toByteArray()).joinToString("") {
|
||||
"%02x".format(it)
|
||||
}
|
||||
}
|
||||
3
packages/decentraland/git/build.gradle.kts
Normal file
3
packages/decentraland/git/build.gradle.kts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
plugins { id("buildlogic.kotlin-library-conventions") }
|
||||
|
||||
dependencies { api(project(":packages:serialize")) }
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package rip.crit.commonip.decentraland.git
|
||||
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
enum class Taint {
|
||||
CLEAN,
|
||||
TAINTED,
|
||||
}
|
||||
|
||||
data class ProvenancedObservation(
|
||||
val location: GeoLocation,
|
||||
val provider: String,
|
||||
val taint: Taint,
|
||||
val evidence: Set<String>,
|
||||
)
|
||||
|
||||
object PromotionPolicy {
|
||||
fun canPublishToCommunity(value: ProvenancedObservation): Boolean =
|
||||
value.taint == Taint.CLEAN && value.evidence.size >= 2
|
||||
|
||||
fun destination(value: ProvenancedObservation): CommonIpRepository =
|
||||
if (canPublishToCommunity(value)) CommonIpRepositories.community
|
||||
else CommonIpRepositories.taintedFull
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package rip.crit.commonip.decentraland.git
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
enum class RepositoryTrust {
|
||||
COMMUNITY,
|
||||
TAINTED,
|
||||
}
|
||||
|
||||
data class CommonIpRepository(
|
||||
val name: String,
|
||||
val trust: RepositoryTrust,
|
||||
val graph: RevisionGraph = RevisionGraph(),
|
||||
)
|
||||
|
||||
object CommonIpRepositories {
|
||||
val community = CommonIpRepository("commonip/community", RepositoryTrust.COMMUNITY)
|
||||
val taintedFull = CommonIpRepository("commonip/tainted-full", RepositoryTrust.TAINTED)
|
||||
}
|
||||
|
||||
data class NodeYaml(
|
||||
val remote: String,
|
||||
val branch: String = "main",
|
||||
val lfs: Boolean = true,
|
||||
val tainted: Boolean = false,
|
||||
val passwd: String? = null,
|
||||
val bootstrap: List<String> = emptyList(),
|
||||
)
|
||||
|
||||
object NodeYamlReader {
|
||||
fun parse(text: String): NodeYaml {
|
||||
val values =
|
||||
text
|
||||
.lineSequence()
|
||||
.map(String::trim)
|
||||
.filter { ':' in it && !it.startsWith('#') }
|
||||
.associate {
|
||||
it.substringBefore(':').trim() to it.substringAfter(':').trim().trim('"', '\'')
|
||||
}
|
||||
val bootstrap = text.lineSequence().map(String::trim).filter { it.startsWith("-") }.map { it.removePrefix("-").trim().trim('"', '\'') }.toList()
|
||||
return NodeYaml(
|
||||
values["remote"] ?: error("node.yaml requires remote"),
|
||||
values["branch"] ?: "main",
|
||||
values["lfs"]?.toBooleanStrictOrNull() ?: true,
|
||||
values["tainted"]?.toBooleanStrictOrNull() ?: false,
|
||||
values["passwd"],
|
||||
bootstrap,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class GitLfsSync(private val repository: Path) {
|
||||
fun sync(config: NodeYaml): Boolean =
|
||||
runCatching {
|
||||
fun command(vararg args: String) =
|
||||
ProcessBuilder(*args)
|
||||
.directory(repository.toFile())
|
||||
.inheritIO()
|
||||
.start()
|
||||
.waitFor() == 0
|
||||
command("git", "remote", "get-url", "origin") ||
|
||||
command("git", "remote", "add", "origin", config.remote)
|
||||
command("git", "fetch", "origin", config.branch) &&
|
||||
(!config.lfs || command("git", "lfs", "pull", "origin", config.branch))
|
||||
}
|
||||
.getOrDefault(false)
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
package rip.crit.commonip.decentraland.git
|
||||
|
||||
import java.time.Instant
|
||||
import java.util.UUID
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
data class GeoCommit(
|
||||
val id: String = UUID.randomUUID().toString(),
|
||||
val parents: Set<String>,
|
||||
val author: String,
|
||||
val observation: GeoLocation,
|
||||
val createdAt: Instant = Instant.now(),
|
||||
)
|
||||
|
||||
enum class MergeState {
|
||||
FAST_FORWARD,
|
||||
MERGED,
|
||||
CONFLICT,
|
||||
}
|
||||
|
||||
data class MergeResult(
|
||||
val state: MergeState,
|
||||
val commit: GeoCommit?,
|
||||
val competing: List<GeoCommit>,
|
||||
val message: String,
|
||||
)
|
||||
|
||||
class RevisionGraph {
|
||||
private val commits = linkedMapOf<String, GeoCommit>()
|
||||
|
||||
fun import(commit: GeoCommit): Boolean {
|
||||
if (commit.parents.any { it !in commits }) return false
|
||||
commits.putIfAbsent(commit.id, commit)
|
||||
return true
|
||||
}
|
||||
|
||||
fun commit(author: String, observation: GeoLocation, parents: Set<String> = heads()): GeoCommit =
|
||||
GeoCommit(parents = parents, author = author, observation = observation).also {
|
||||
commits[it.id] = it
|
||||
}
|
||||
|
||||
fun heads(): Set<String> = commits.keys - commits.values.flatMap { it.parents }.toSet()
|
||||
|
||||
fun merge(left: String, right: String, author: String): MergeResult {
|
||||
val a =
|
||||
commits[left]
|
||||
?: return MergeResult(
|
||||
MergeState.CONFLICT,
|
||||
null,
|
||||
emptyList(),
|
||||
"left revision is unknown",
|
||||
)
|
||||
val b =
|
||||
commits[right]
|
||||
?: return MergeResult(
|
||||
MergeState.CONFLICT,
|
||||
null,
|
||||
emptyList(),
|
||||
"right revision is unknown",
|
||||
)
|
||||
if (isAncestor(a.id, b.id))
|
||||
return MergeResult(MergeState.FAST_FORWARD, b, emptyList(), "left fast-forwards to right")
|
||||
if (isAncestor(b.id, a.id))
|
||||
return MergeResult(MergeState.FAST_FORWARD, a, emptyList(), "right fast-forwards to left")
|
||||
if (!compatible(a.observation, b.observation))
|
||||
return MergeResult(
|
||||
MergeState.CONFLICT,
|
||||
null,
|
||||
listOf(a, b),
|
||||
"country or coordinates disagree, proof-of-location required",
|
||||
)
|
||||
val winner = listOf(a, b).maxBy { it.observation.confidence }
|
||||
val merged =
|
||||
GeoCommit(
|
||||
parents = setOf(a.id, b.id),
|
||||
author = author,
|
||||
observation =
|
||||
winner.observation.copy(
|
||||
source = "merge:$author",
|
||||
confidence = (a.observation.confidence + b.observation.confidence) / 2,
|
||||
),
|
||||
)
|
||||
commits[merged.id] = merged
|
||||
return MergeResult(MergeState.MERGED, merged, emptyList(), "two verified revisions merged")
|
||||
}
|
||||
|
||||
private fun isAncestor(ancestor: String, descendant: String): Boolean =
|
||||
commits[descendant]?.parents?.let { parents ->
|
||||
ancestor in parents || parents.any { isAncestor(ancestor, it) }
|
||||
} ?: false
|
||||
|
||||
private fun compatible(a: GeoLocation, b: GeoLocation): Boolean {
|
||||
val aLat = a.latitude
|
||||
val bLat = b.latitude
|
||||
return a.prefix == b.prefix &&
|
||||
a.countryCode == b.countryCode &&
|
||||
(aLat == null || bLat == null || kotlin.math.abs(aLat - bLat) < 1.0)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
package rip.crit.commonip.decentraland.git
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.security.PrivateKey
|
||||
import java.security.PublicKey
|
||||
import java.security.Signature
|
||||
import java.time.Instant
|
||||
import java.util.Base64
|
||||
|
||||
data class SignedGeoCommit(val revision: GeoCommit, val signature: String) {
|
||||
fun payload(): ByteArray =
|
||||
listOf(
|
||||
revision.id,
|
||||
revision.parents.sorted().joinToString(","),
|
||||
revision.author,
|
||||
revision.observation.prefix,
|
||||
revision.observation.countryCode,
|
||||
revision.observation.latitude,
|
||||
revision.observation.longitude,
|
||||
revision.createdAt,
|
||||
)
|
||||
.joinToString("|")
|
||||
.toByteArray(StandardCharsets.UTF_8)
|
||||
}
|
||||
|
||||
object CommitSigner {
|
||||
fun sign(revision: GeoCommit, key: PrivateKey): SignedGeoCommit {
|
||||
val unsigned = SignedGeoCommit(revision, "")
|
||||
val signature =
|
||||
Signature.getInstance("Ed25519")
|
||||
.apply {
|
||||
initSign(key)
|
||||
update(unsigned.payload())
|
||||
}
|
||||
.sign()
|
||||
return unsigned.copy(signature = Base64.getEncoder().encodeToString(signature))
|
||||
}
|
||||
|
||||
fun verify(commit: SignedGeoCommit, key: PublicKey): Boolean =
|
||||
runCatching {
|
||||
Signature.getInstance("Ed25519")
|
||||
.apply {
|
||||
initVerify(key)
|
||||
update(commit.payload())
|
||||
}
|
||||
.verify(Base64.getDecoder().decode(commit.signature))
|
||||
}
|
||||
.getOrDefault(false)
|
||||
}
|
||||
|
||||
data class ProofOfLocation(
|
||||
val revisionId: String,
|
||||
val observer: String,
|
||||
val observedAt: Instant,
|
||||
val reachable: Boolean,
|
||||
val latencyMs: Long?,
|
||||
val signature: String,
|
||||
)
|
||||
|
||||
class SignedCommitLedger(
|
||||
private val graph: RevisionGraph,
|
||||
private val keys: Map<String, PublicKey>,
|
||||
) {
|
||||
fun accept(commit: SignedGeoCommit): Boolean =
|
||||
(keys[commit.revision.author]?.let { CommitSigner.verify(commit, it) } ?: false) &&
|
||||
graph.import(commit.revision)
|
||||
|
||||
fun resolve(left: SignedGeoCommit, right: SignedGeoCommit, author: String): MergeResult {
|
||||
require(accept(left) && accept(right)) { "all merge inputs must have valid signatures" }
|
||||
return graph.merge(left.revision.id, right.revision.id, author)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package rip.crit.commonip.decentraland.git
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
class RepositoryTest {
|
||||
@Test
|
||||
fun `node yaml retains trust and password fields`() {
|
||||
val config = NodeYamlReader.parse("remote: ssh://example/repo\ntainted: true\npasswd: secret\nbootstrap:\n - peer.example:7400")
|
||||
assertTrue(config.tainted)
|
||||
assertEquals("secret", config.passwd)
|
||||
assertEquals(listOf("peer.example:7400"), config.bootstrap)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `community promotion needs clean independent evidence`() {
|
||||
val location = GeoLocation("192.0.2.0/24", "US")
|
||||
assertFalse(
|
||||
PromotionPolicy.canPublishToCommunity(
|
||||
ProvenancedObservation(location, "test", Taint.CLEAN, setOf("one"))
|
||||
)
|
||||
)
|
||||
assertTrue(
|
||||
PromotionPolicy.canPublishToCommunity(
|
||||
ProvenancedObservation(location, "test", Taint.CLEAN, setOf("one", "two"))
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
7
packages/decentraland/kademlia/build.gradle.kts
Normal file
7
packages/decentraland/kademlia/build.gradle.kts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
plugins { id("buildlogic.kotlin-library-conventions") }
|
||||
|
||||
dependencies {
|
||||
api(project(":packages:serialize"))
|
||||
implementation("io.ep2p:kademlia-api:5.1.2-RELEASE")
|
||||
implementation("io.ep2p:kademlia-netty:0.3.2-RELEASE")
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
package rip.crit.commonip.decentraland.kademlia
|
||||
|
||||
import java.net.DatagramPacket
|
||||
import java.net.DatagramSocket
|
||||
import java.net.InetSocketAddress
|
||||
import java.security.MessageDigest
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import kotlin.concurrent.thread
|
||||
|
||||
data class KademliaPeer(val id: String, val address: InetSocketAddress)
|
||||
|
||||
class KademliaNode(private val bind: InetSocketAddress, private val registry: NodeRegistry) :
|
||||
AutoCloseable {
|
||||
val id = sha256("${bind.hostString}:${bind.port}")
|
||||
private val namer = NodeNamer()
|
||||
private val peers = ConcurrentHashMap<String, KademliaPeer>()
|
||||
private val values = ConcurrentHashMap<String, String>()
|
||||
private val socket = DatagramSocket(bind)
|
||||
private val lastOutbound = ConcurrentHashMap<String, Long>()
|
||||
@Volatile private var running = true
|
||||
|
||||
fun start() =
|
||||
thread(name = "commonip-kademlia-$id", isDaemon = true) {
|
||||
registry.register(
|
||||
NetworkNode(
|
||||
namer.name(id),
|
||||
socket.localSocketAddress.toString(),
|
||||
NodeState.ONLINE,
|
||||
0,
|
||||
values.size,
|
||||
)
|
||||
)
|
||||
val bytes = ByteArray(8192)
|
||||
while (running) runCatching {
|
||||
val packet = DatagramPacket(bytes, bytes.size)
|
||||
socket.receive(packet)
|
||||
handle(packet)
|
||||
}
|
||||
}
|
||||
|
||||
fun bootstrap(seed: InetSocketAddress) {
|
||||
if (peers.values.none { it.address == seed }) send(seed, "PING|$id|${bind.hostString}|${socket.localPort}")
|
||||
}
|
||||
|
||||
fun store(key: String, value: String) {
|
||||
values[key] = value
|
||||
closest(key).forEach { send(it.address, "STORE|$id|$key|${value.encodeBase64()}") }
|
||||
}
|
||||
|
||||
fun findValue(key: String): String? = values[key]
|
||||
|
||||
fun closest(key: String, count: Int = 20): List<KademliaPeer> =
|
||||
peers.values.sortedBy { xorDistance(it.id, key) }.take(count)
|
||||
|
||||
private fun handle(packet: DatagramPacket) {
|
||||
val data =
|
||||
packet.data
|
||||
.copyOfRange(packet.offset, packet.offset + packet.length)
|
||||
.decodeToString()
|
||||
.split('|', limit = 5)
|
||||
val remote = InetSocketAddress(packet.address, packet.port)
|
||||
when (data.firstOrNull()) {
|
||||
"PING" -> {
|
||||
peers[data[1]] = KademliaPeer(data[1], InetSocketAddress(data[2], data[3].toInt()))
|
||||
val name = namer.name(data[1])
|
||||
registry.register(NetworkNode(name, remote.toString(), NodeState.ONLINE, null, 0))
|
||||
registry.publish(name, "kademlia ping", remote.toString())
|
||||
send(remote, "PONG|$id")
|
||||
}
|
||||
"PONG" -> {
|
||||
peers[data[1]] = KademliaPeer(data[1], remote)
|
||||
val name = namer.name(data[1])
|
||||
registry.register(NetworkNode(name, remote.toString(), NodeState.ONLINE, null, 0))
|
||||
registry.publish(name, "kademlia bootstrap confirmed", remote.toString())
|
||||
}
|
||||
"STORE" -> values[data[2]] = data[3].decodeBase64()
|
||||
"FIND_VALUE" ->
|
||||
values[data[2]]?.let { send(remote, "VALUE|$id|${data[2]}|${it.encodeBase64()}") }
|
||||
"FIND_NODE" ->
|
||||
send(
|
||||
remote,
|
||||
"NODES|$id|${closest(data[2]).joinToString(";") { "${it.id}@${it.address.hostString}:${it.address.port}" }}",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun send(target: InetSocketAddress, message: String) {
|
||||
val key = "${target.hostString}:${target.port}:${message.substringBefore('|')}"
|
||||
val previous = lastOutbound.put(key, System.currentTimeMillis()) ?: 0
|
||||
if (System.currentTimeMillis() - previous < 500) return
|
||||
socket.send(DatagramPacket(message.encodeToByteArray(), message.length, target))
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
running = false
|
||||
socket.close()
|
||||
}
|
||||
|
||||
private fun sha256(value: String) =
|
||||
MessageDigest.getInstance("SHA-256").digest(value.encodeToByteArray()).joinToString("") {
|
||||
"%02x".format(it)
|
||||
}
|
||||
|
||||
private fun xorDistance(left: String, right: String) =
|
||||
left.zip(right.padEnd(left.length, '0')).count { it.first != it.second }
|
||||
|
||||
private fun String.encodeBase64() =
|
||||
java.util.Base64.getUrlEncoder().encodeToString(encodeToByteArray())
|
||||
|
||||
private fun String.decodeBase64() = java.util.Base64.getUrlDecoder().decode(this).decodeToString()
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package rip.crit.commonip.decentraland.kademlia
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
class NodeNamer {
|
||||
private val votes = ConcurrentHashMap<String, ConcurrentHashMap<String, MutableSet<String>>>()
|
||||
|
||||
fun observe(nodeId: String, observerId: String, alias: String) {
|
||||
votes
|
||||
.getOrPut(nodeId) { ConcurrentHashMap() }
|
||||
.getOrPut(alias) { ConcurrentHashMap.newKeySet() }
|
||||
.add(observerId)
|
||||
}
|
||||
|
||||
fun name(nodeId: String): String =
|
||||
votes[nodeId]?.maxByOrNull { it.value.size }?.takeIf { it.value.size >= 2 }?.key
|
||||
?: deterministic(nodeId)
|
||||
|
||||
private fun deterministic(id: String): String {
|
||||
val adjective = arrayOf("amber", "cedar", "delta", "ember", "flint", "grove", "harbor", "iris")
|
||||
val noun = arrayOf("atlas", "beacon", "cinder", "drift", "echo", "forge", "glade", "horizon")
|
||||
val number = id.take(8).toLong(16)
|
||||
return "${adjective[(number % adjective.size).toInt()]}-${noun[((number / adjective.size) % noun.size).toInt()]}-${id.takeLast(4)}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package rip.crit.commonip.decentraland.kademlia
|
||||
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
enum class NodeState {
|
||||
ONLINE,
|
||||
DEGRADED,
|
||||
OFFLINE,
|
||||
}
|
||||
|
||||
data class NetworkNode(
|
||||
val id: String,
|
||||
val endpoint: String,
|
||||
val state: NodeState,
|
||||
val latencyMs: Long?,
|
||||
val records: Int,
|
||||
val lastSeen: Instant = Instant.now(),
|
||||
)
|
||||
|
||||
data class NetworkActivity(
|
||||
val at: Instant,
|
||||
val nodeId: String,
|
||||
val action: String,
|
||||
val detail: String,
|
||||
)
|
||||
|
||||
class NodeRegistry {
|
||||
private val nodes = linkedMapOf<String, NetworkNode>()
|
||||
private val activity = CopyOnWriteArrayList<NetworkActivity>()
|
||||
|
||||
@Synchronized
|
||||
fun register(node: NetworkNode) {
|
||||
nodes[node.id] = node
|
||||
publish(node.id, "peer announced", "${node.endpoint} is ${node.state.name.lowercase()}")
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun update(id: String, state: NodeState, latencyMs: Long? = null, records: Int? = null) {
|
||||
val prior = nodes[id] ?: return
|
||||
nodes[id] =
|
||||
prior.copy(
|
||||
state = state,
|
||||
latencyMs = latencyMs ?: prior.latencyMs,
|
||||
records = records ?: prior.records,
|
||||
lastSeen = Instant.now(),
|
||||
)
|
||||
publish(id, "status changed", state.name.lowercase())
|
||||
}
|
||||
|
||||
fun snapshot(): List<NetworkNode> = synchronized(this) { nodes.values.sortedBy { it.id } }
|
||||
|
||||
fun feed(limit: Int = 12): List<NetworkActivity> = activity.takeLast(limit).reversed()
|
||||
|
||||
fun publish(nodeId: String, action: String, detail: String) {
|
||||
activity += NetworkActivity(Instant.now(), nodeId, action, detail)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package rip.crit.commonip.decentraland.kademlia
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class NodeNamerTest {
|
||||
@Test
|
||||
fun `two peer votes override deterministic node name`() {
|
||||
val namer = NodeNamer()
|
||||
namer.observe("abcdef012345", "peer-a", "north")
|
||||
namer.observe("abcdef012345", "peer-b", "north")
|
||||
assertEquals("north", namer.name("abcdef012345"))
|
||||
}
|
||||
}
|
||||
3
packages/decentraland/settings.gradle.kts
Normal file
3
packages/decentraland/settings.gradle.kts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
rootProject.name = "commonip-decentraland"
|
||||
|
||||
include("kademlia", "git", "chksum")
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package rip.crit.commonip.decentraland
|
||||
|
||||
import java.nio.file.Path
|
||||
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory
|
||||
import org.apache.sshd.server.SshServer
|
||||
import org.apache.sshd.server.auth.password.PasswordAuthenticator
|
||||
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider
|
||||
|
||||
class SshDashboardService(
|
||||
port: Int,
|
||||
hostKey: Path,
|
||||
username: String,
|
||||
password: CharArray,
|
||||
private val dashboard: () -> String,
|
||||
) : AutoCloseable {
|
||||
private val server =
|
||||
SshServer.setUpDefaultServer().apply {
|
||||
this.port = port
|
||||
keyPairProvider = SimpleGeneratorHostKeyProvider(hostKey)
|
||||
passwordAuthenticator = PasswordAuthenticator { user, candidate, _ ->
|
||||
user == username && candidate.toCharArray().contentEquals(password)
|
||||
}
|
||||
fileSystemFactory = VirtualFileSystemFactory()
|
||||
}
|
||||
|
||||
fun start() {
|
||||
server.start()
|
||||
}
|
||||
|
||||
fun render(): String = dashboard()
|
||||
|
||||
override fun close() = server.close()
|
||||
}
|
||||
13
packages/ml/build.gradle.kts
Normal file
13
packages/ml/build.gradle.kts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-library-conventions")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api(project(":packages:serialize"))
|
||||
}
|
||||
|
|
@ -0,0 +1,235 @@
|
|||
package rip.crit.commonip.ml
|
||||
|
||||
import kotlin.math.*
|
||||
|
||||
data class GeoLocation(
|
||||
val latitude: Double?,
|
||||
val longitude: Double?,
|
||||
val confidence: Double = 1.0,
|
||||
)
|
||||
|
||||
data class LocationEstimate(
|
||||
val latitude: Double,
|
||||
val longitude: Double,
|
||||
val confidence: Double,
|
||||
val sampleCount: Int,
|
||||
val discardedCount: Int,
|
||||
val spatialSpreadKm: Double,
|
||||
)
|
||||
|
||||
data class Vector3D(val x: Double, val y: Double, val z: Double) {
|
||||
fun normalize(): Vector3D {
|
||||
val mag = sqrt(x * x + y * y + z * z)
|
||||
return if (mag == 0.0) Vector3D(0.0, 0.0, 0.0) else Vector3D(x / mag, y / mag, z / mag)
|
||||
}
|
||||
|
||||
fun toGeoLocation(confidence: Double = 1.0): GeoLocation {
|
||||
val norm = this.normalize()
|
||||
val lat = Math.toDegrees(asin(norm.z.coerceIn(-1.0, 1.0)))
|
||||
val lon = Math.toDegrees(atan2(norm.y, norm.x))
|
||||
return GeoLocation(lat, lon, confidence)
|
||||
}
|
||||
}
|
||||
|
||||
fun GeoLocation.toVector3D(): Vector3D {
|
||||
val latRad = Math.toRadians(this.latitude!!)
|
||||
val lonRad = Math.toRadians(this.longitude!!)
|
||||
return Vector3D(
|
||||
x = cos(latRad) * cos(lonRad),
|
||||
y = cos(latRad) * sin(lonRad),
|
||||
z = sin(latRad),
|
||||
)
|
||||
}
|
||||
|
||||
class LocationEstimator(
|
||||
private val dbscanEpsKm: Double = 250.0,
|
||||
private val dbscanMinPts: Int = 2,
|
||||
private val medianIterations: Int = 20,
|
||||
) {
|
||||
|
||||
fun estimate(observations: List<GeoLocation>): LocationEstimate? {
|
||||
val validPoints = observations.filter { it.latitude != null && it.longitude != null }
|
||||
if (validPoints.isEmpty()) return null
|
||||
|
||||
val clusteredPoints = dbscanCluster(validPoints, dbscanEpsKm, dbscanMinPts)
|
||||
val selected = clusteredPoints.ifEmpty { validPoints }
|
||||
|
||||
val (estLat, estLon) = computeGeometricMedian(selected, medianIterations)
|
||||
|
||||
val distances = selected.map { distanceKm(estLat, estLon, it.latitude!!, it.longitude!!) }
|
||||
val standardDistanceKm = sqrt(distances.sumOf { it * it } / selected.size)
|
||||
|
||||
val meanSampleConfidence = selected.map { max(it.confidence, 0.01) }.average()
|
||||
val densityFactor = 1.0 / (1.0 + exp(-(selected.size - 3.0) / 2.0))
|
||||
val spreadPenalty = 1.0 / (1.0 + (standardDistanceKm / 300.0))
|
||||
|
||||
val overallConfidence =
|
||||
(meanSampleConfidence * densityFactor * spreadPenalty).coerceIn(0.0, 1.0)
|
||||
|
||||
return LocationEstimate(
|
||||
latitude = estLat,
|
||||
longitude = estLon,
|
||||
confidence = overallConfidence,
|
||||
sampleCount = selected.size,
|
||||
discardedCount = validPoints.size - selected.size,
|
||||
spatialSpreadKm = standardDistanceKm,
|
||||
)
|
||||
}
|
||||
|
||||
/** Finds the primary spatial cluster using DBSCAN logic. */
|
||||
private fun dbscanCluster(
|
||||
points: List<GeoLocation>,
|
||||
epsKm: Double,
|
||||
minPts: Int,
|
||||
): List<GeoLocation> {
|
||||
val visited = BooleanArray(points.size)
|
||||
val clusters = mutableListOf<MutableList<GeoLocation>>()
|
||||
|
||||
for (i in points.indices) {
|
||||
if (visited[i]) continue
|
||||
visited[i] = true
|
||||
|
||||
val neighbors = getNeighbors(points, i, epsKm)
|
||||
if (neighbors.size >= minPts) {
|
||||
val currentCluster = mutableListOf(points[i])
|
||||
val queue = neighbors.toMutableList()
|
||||
var qIndex = 0
|
||||
|
||||
while (qIndex < queue.size) {
|
||||
val pIndex = queue[qIndex++]
|
||||
if (!visited[pIndex]) {
|
||||
visited[pIndex] = true
|
||||
val subNeighbors = getNeighbors(points, pIndex, epsKm)
|
||||
if (subNeighbors.size >= minPts) {
|
||||
queue.addAll(subNeighbors.filter { !queue.contains(it) })
|
||||
}
|
||||
}
|
||||
if (!currentCluster.contains(points[pIndex])) {
|
||||
currentCluster.add(points[pIndex])
|
||||
}
|
||||
}
|
||||
clusters.add(currentCluster)
|
||||
}
|
||||
}
|
||||
|
||||
return clusters.maxByOrNull { cluster -> cluster.sumOf { max(it.confidence, 0.05) } }
|
||||
?: emptyList()
|
||||
}
|
||||
|
||||
private fun getNeighbors(points: List<GeoLocation>, centerIdx: Int, epsKm: Double): List<Int> {
|
||||
val center = points[centerIdx]
|
||||
val result = mutableListOf<Int>()
|
||||
for (i in points.indices) {
|
||||
if (
|
||||
distanceKm(
|
||||
center.latitude!!,
|
||||
center.longitude!!,
|
||||
points[i].latitude!!,
|
||||
points[i].longitude!!,
|
||||
) <= epsKm
|
||||
) {
|
||||
result.add(i)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Weiszfeld's algorithm for finding the spatial geometric median (L1 median). */
|
||||
private fun computeGeometricMedian(
|
||||
points: List<GeoLocation>,
|
||||
iterations: Int,
|
||||
): Pair<Double, Double> {
|
||||
if (points.size == 1) return Pair(points[0].latitude!!, points[0].longitude!!)
|
||||
|
||||
val weights = points.map { max(it.confidence, 0.05) }
|
||||
val sumWeight = weights.sum()
|
||||
|
||||
var currentVector =
|
||||
points
|
||||
.zip(weights)
|
||||
.map { (pt, w) ->
|
||||
val vec = pt.toVector3D()
|
||||
Vector3D(vec.x * w, vec.y * w, vec.z * w)
|
||||
}
|
||||
.reduce { acc, v ->
|
||||
Vector3D(acc.x + v.x, acc.y + v.y, acc.z + v.z)
|
||||
}
|
||||
.let { Vector3D(it.x / sumWeight, it.y / sumWeight, it.z / sumWeight).normalize() }
|
||||
|
||||
var currentGeo = currentVector.toGeoLocation()
|
||||
|
||||
repeat(iterations) {
|
||||
var sumX = 0.0
|
||||
var sumY = 0.0
|
||||
var sumZ = 0.0
|
||||
var totalW = 0.0
|
||||
|
||||
for (i in points.indices) {
|
||||
val ptVec = points[i].toVector3D()
|
||||
val dist =
|
||||
max(
|
||||
distanceKm(
|
||||
currentGeo.latitude!!,
|
||||
currentGeo.longitude!!,
|
||||
points[i].latitude!!,
|
||||
points[i].longitude!!,
|
||||
),
|
||||
0.1,
|
||||
)
|
||||
val invWeight = weights[i] / dist
|
||||
|
||||
sumX += ptVec.x * invWeight
|
||||
sumY += ptVec.y * invWeight
|
||||
sumZ += ptVec.z * invWeight
|
||||
totalW += invWeight
|
||||
}
|
||||
|
||||
if (totalW > 0.0) {
|
||||
currentVector = Vector3D(sumX / totalW, sumY / totalW, sumZ / totalW).normalize()
|
||||
currentGeo = currentVector.toGeoLocation()
|
||||
}
|
||||
}
|
||||
|
||||
return Pair(currentGeo.latitude!!, currentGeo.longitude!!)
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun distanceKm(aLat: Double, aLon: Double, bLat: Double, bLon: Double): Double {
|
||||
val dLat = Math.toRadians(bLat - aLat)
|
||||
val dLon = Math.toRadians(bLon - aLon)
|
||||
val h =
|
||||
sin(dLat / 2).pow(2) +
|
||||
cos(Math.toRadians(aLat)) * cos(Math.toRadians(bLat)) * sin(dLon / 2).pow(2)
|
||||
return 6371.0 * 2 * atan2(sqrt(h), sqrt(1 - h))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data class NodeTarget(
|
||||
val nodeId: String,
|
||||
val capacity: Int,
|
||||
val latitude: Double?,
|
||||
val longitude: Double?,
|
||||
)
|
||||
|
||||
object LocationDispatcher {
|
||||
fun select(
|
||||
nodes: List<NodeTarget>,
|
||||
estimate: LocationEstimate,
|
||||
count: Int = 3,
|
||||
): List<NodeTarget> =
|
||||
nodes
|
||||
.filter { it.capacity > 0 && it.latitude != null && it.longitude != null }
|
||||
.sortedBy { node ->
|
||||
val distance =
|
||||
LocationEstimator.distanceKm(
|
||||
estimate.latitude,
|
||||
estimate.longitude,
|
||||
node.latitude!!,
|
||||
node.longitude!!,
|
||||
)
|
||||
|
||||
distance / sqrt(node.capacity.toDouble())
|
||||
}
|
||||
.take(count)
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package rip.crit.commonip.ml
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class LocationEstimatorTest {
|
||||
@Test
|
||||
fun `estimator rejects distant outlier`() {
|
||||
val result =
|
||||
LocationEstimator()
|
||||
.estimate(
|
||||
listOf(
|
||||
GeoLocation(40.0, -75.0, 0.9),
|
||||
GeoLocation(40.1, -75.1, 0.9),
|
||||
GeoLocation(-33.0, 151.0, 0.9),
|
||||
)
|
||||
)!!
|
||||
assertEquals(1, result.discardedCount)
|
||||
}
|
||||
}
|
||||
13
packages/rawr/build.gradle.kts
Normal file
13
packages/rawr/build.gradle.kts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-library-conventions")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api(project(":packages:serialize"))
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package rip.crit.commonip.rawr.feeds
|
||||
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||
|
||||
object RFC8805 {
|
||||
fun parse(csv: String): GeoFeed = GeoFeedCodec.decode(csv, GeoFeedFormat.CSV)
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package rip.crit.commonip.rawr.feeds
|
||||
|
||||
data class RFC9632(
|
||||
val feedUrl: String,
|
||||
val signatureUrl: String?,
|
||||
val valid: Boolean,
|
||||
val signer: String? = null,
|
||||
) {
|
||||
companion object {
|
||||
fun fromDnsRecord(record: String): RFC9632? {
|
||||
val values =
|
||||
record
|
||||
.split(';')
|
||||
.map { it.trim() }
|
||||
.associate { it.substringBefore('=').lowercase() to it.substringAfter('=', "") }
|
||||
val url = values["geofeed"] ?: return null
|
||||
return RFC9632(url, values["sig"], values["valid"] != "false", values["signer"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package rip.crit.commonip.rawr.feeds
|
||||
|
||||
import rip.crit.commonip.rawr.types.Ingest
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
class RirFeed(private val name: String, private val loader: () -> String) : Ingest {
|
||||
override fun ingest(): GeoFeed =
|
||||
GeoFeedCodec.decode(loader(), GeoFeedFormat.CSV).let { feed ->
|
||||
feed.copy(records = feed.records.map { it.copy(source = name) })
|
||||
}
|
||||
}
|
||||
|
||||
object DelegatedStats {
|
||||
fun parse(payload: String, source: String): GeoFeed =
|
||||
GeoFeed(
|
||||
payload
|
||||
.lineSequence()
|
||||
.filter { it.isNotBlank() && !it.startsWith('#') && !it.startsWith("2|") }
|
||||
.mapNotNull { line ->
|
||||
val f = line.split('|')
|
||||
if (f.size < 5 || f[2].lowercase() !in setOf("ipv4", "ipv6") || f[1] == "*") null
|
||||
else {
|
||||
val prefix =
|
||||
if (f[2].lowercase() == "ipv4") "${f[3]}/${prefixFromCount(f[4])}"
|
||||
else "${f[3]}/${f.getOrElse(4) { "48" }}"
|
||||
GeoLocation(prefix, f[1].uppercase(), source = source, confidence = 0.25)
|
||||
}
|
||||
}
|
||||
.toList()
|
||||
)
|
||||
|
||||
private fun prefixFromCount(count: String): Int =
|
||||
32 - Integer.numberOfTrailingZeros(count.toIntOrNull() ?: 1)
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
class AFRINIC(loader: (() -> String)? = null) :
|
||||
RirProvider(
|
||||
"AFRINIC",
|
||||
"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
|
||||
loader,
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
class APNIC(loader: (() -> String)? = null) :
|
||||
RirProvider(
|
||||
"APNIC",
|
||||
"https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest",
|
||||
loader,
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
class ARINIngest(loader: (() -> String)? = null) :
|
||||
RirProvider(
|
||||
"ARIN",
|
||||
"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
|
||||
loader,
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
class LACNIC(loader: (() -> String)? = null) :
|
||||
RirProvider(
|
||||
"LACNIC",
|
||||
"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
|
||||
loader,
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
class RIPENCC(loader: (() -> String)? = null) :
|
||||
RirProvider(
|
||||
"RIPE NCC",
|
||||
"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
|
||||
loader,
|
||||
)
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
import java.net.URI
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||
|
||||
enum class RirMetadataField(val keys: Set<String>) {
|
||||
REMARKS(setOf("remarks", "remark")),
|
||||
COMMENT(setOf("comment", "comments")),
|
||||
}
|
||||
|
||||
class RirGeoFeedLoader(
|
||||
private val source: String,
|
||||
private val acceptedFields: Set<RirMetadataField>,
|
||||
private val fetch: (String) -> String = RirGeoFeedLoader::httpGet,
|
||||
) {
|
||||
fun urls(registryObject: String): Set<String> =
|
||||
registryObject
|
||||
.lineSequence()
|
||||
.map(String::trim)
|
||||
.filter { ':' in it }
|
||||
.filter { line ->
|
||||
acceptedFields.any { field ->
|
||||
line.substringBefore(':').trim().lowercase() in field.keys
|
||||
}
|
||||
}
|
||||
.flatMap { urlPattern.findAll(it).map { match -> match.value } }
|
||||
.toSet()
|
||||
|
||||
fun pull(registryObject: String): List<GeoFeed> =
|
||||
urls(registryObject).map { url -> GeoFeedCodec.decode(fetch(url), GeoFeedFormat.CSV) }
|
||||
|
||||
private companion object {
|
||||
val urlPattern = Regex("https?://[^\\s,;\"]+")
|
||||
val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
|
||||
|
||||
fun httpGet(url: String): String =
|
||||
client
|
||||
.send(
|
||||
HttpRequest.newBuilder(URI(url)).GET().build(),
|
||||
HttpResponse.BodyHandlers.ofString(),
|
||||
)
|
||||
.body()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
import java.net.URI
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
import rip.crit.commonip.rawr.feeds.DelegatedStats
|
||||
import rip.crit.commonip.rawr.types.Ingest
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
|
||||
abstract class RirProvider(
|
||||
private val name: String,
|
||||
private val endpoint: String,
|
||||
private val loader: (() -> String)? = null,
|
||||
) : Ingest {
|
||||
override fun ingest(): GeoFeed =
|
||||
DelegatedStats.parse(
|
||||
loader?.invoke()
|
||||
?: client
|
||||
.send(
|
||||
HttpRequest.newBuilder(URI(endpoint)).GET().build(),
|
||||
HttpResponse.BodyHandlers.ofString(),
|
||||
)
|
||||
.body(),
|
||||
name,
|
||||
)
|
||||
|
||||
private companion object {
|
||||
val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package rip.crit.commonip.rawr.types
|
||||
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
interface Ingest {
|
||||
fun ingest(): GeoFeed
|
||||
|
||||
fun lookupOne(address: String): GeoLocation? =
|
||||
ingest()
|
||||
.records
|
||||
.filter { Cidr.contains(it.prefix, address) }
|
||||
.maxByOrNull { Cidr.prefixLength(it.prefix) }
|
||||
}
|
||||
|
||||
object Cidr {
|
||||
fun prefixLength(cidr: String): Int = cidr.substringAfter('/').toIntOrNull() ?: -1
|
||||
|
||||
fun contains(cidr: String, address: String): Boolean {
|
||||
val parts = cidr.split('/', limit = 2)
|
||||
val bits = parts.getOrNull(1)?.toIntOrNull() ?: return false
|
||||
if (bits !in 0..32) return false
|
||||
val network = ipv4(parts[0]) ?: return false
|
||||
val target = ipv4(address) ?: return false
|
||||
val mask = if (bits == 0) 0 else (-1 shl (32 - bits))
|
||||
return network and mask == target and mask
|
||||
}
|
||||
|
||||
private fun ipv4(value: String): Int? {
|
||||
val fields = value.split('.')
|
||||
if (fields.size != 4) return null
|
||||
return fields.fold(0) { result, field ->
|
||||
field.toIntOrNull()?.takeIf { it in 0..255 }?.let { (result shl 8) or it } ?: return null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package rip.crit.commonip.rawr.providers
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class RirProviderTest {
|
||||
@Test
|
||||
fun `arin parses delegated ipv4 fixture`() {
|
||||
val feed = ARINIngest { "arin|US|ipv4|192.0.2.0|256|20260101|allocated" }.ingest()
|
||||
assertEquals("192.0.2.0/24", feed.records.single().prefix)
|
||||
assertEquals("US", feed.records.single().countryCode)
|
||||
}
|
||||
}
|
||||
9
packages/serialize/build.gradle.kts
Normal file
9
packages/serialize/build.gradle.kts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/*
|
||||
* This file was generated by the Gradle 'init' task.
|
||||
*
|
||||
* This project uses @Incubating APIs which are subject to change.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-library-conventions")
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
package rip.crit.commonip.serialize
|
||||
|
||||
fun String.deserializeGeoFeed(format: GeoFeedFormat): GeoFeed = GeoFeedCodec.decode(this, format)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package rip.crit.commonip.serialize
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
data class GeoLocation(
|
||||
val prefix: String,
|
||||
val countryCode: String,
|
||||
val region: String? = null,
|
||||
val city: String? = null,
|
||||
val postalCode: String? = null,
|
||||
val latitude: Double? = null,
|
||||
val longitude: Double? = null,
|
||||
val source: String = "unknown",
|
||||
val observedAt: Instant = Instant.now(),
|
||||
val confidence: Double = 0.5,
|
||||
) {
|
||||
init {
|
||||
require('/' in prefix) { "prefix must be CIDR notation" }
|
||||
require(countryCode.length == 2) { "countryCode must be ISO 3166-1 alpha-2" }
|
||||
require(confidence in 0.0..1.0) { "confidence must be between 0 and 1" }
|
||||
}
|
||||
}
|
||||
|
||||
data class GeoFeed(
|
||||
val records: List<GeoLocation>,
|
||||
val generatedAt: Instant = Instant.now(),
|
||||
val schemaVersion: String = "commonip/geofeed/v1",
|
||||
)
|
||||
|
||||
enum class GeoFeedFormat {
|
||||
CSV,
|
||||
JSON,
|
||||
XML,
|
||||
}
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
package rip.crit.commonip.serialize
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
object GeoFeedCodec {
|
||||
fun encode(feed: GeoFeed, format: GeoFeedFormat): String =
|
||||
when (format) {
|
||||
GeoFeedFormat.CSV ->
|
||||
feed.records.joinToString("\n") { record ->
|
||||
listOf(
|
||||
record.prefix,
|
||||
record.countryCode,
|
||||
record.region.orEmpty(),
|
||||
record.city.orEmpty(),
|
||||
record.postalCode.orEmpty(),
|
||||
)
|
||||
.joinToString(",") { csv(it) }
|
||||
}
|
||||
GeoFeedFormat.JSON ->
|
||||
"{\"schema\":\"${feed.schemaVersion}\",\"generatedAt\":\"${feed.generatedAt}\",\"records\":[" +
|
||||
feed.records.joinToString(",") { jsonRecord(it) } +
|
||||
"]}"
|
||||
GeoFeedFormat.XML ->
|
||||
"<geofeed schema=\"${xml(feed.schemaVersion)}\" generatedAt=\"${feed.generatedAt}\">" +
|
||||
feed.records.joinToString("") { record ->
|
||||
"<location prefix=\"${xml(record.prefix)}\" country=\"${xml(record.countryCode)}\"${attr("region", record.region)}${attr("city", record.city)}${attr("postalCode", record.postalCode)}${attr("latitude", record.latitude)}${attr("longitude", record.longitude)} source=\"${xml(record.source)}\" observedAt=\"${record.observedAt}\" confidence=\"${record.confidence}\"/>"
|
||||
} +
|
||||
"</geofeed>"
|
||||
}
|
||||
|
||||
fun decode(payload: String, format: GeoFeedFormat): GeoFeed =
|
||||
when (format) {
|
||||
GeoFeedFormat.CSV ->
|
||||
GeoFeed(
|
||||
payload
|
||||
.lineSequence()
|
||||
.filter { it.isNotBlank() && !it.trimStart().startsWith("#") }
|
||||
.map { csvRecord(it) }
|
||||
.toList()
|
||||
)
|
||||
GeoFeedFormat.JSON ->
|
||||
GeoFeed(
|
||||
jsonObjects(
|
||||
payload.substringAfter("\"records\":[", "").substringBeforeLast("]", "")
|
||||
)
|
||||
.map { jsonToRecord(it) }
|
||||
)
|
||||
GeoFeedFormat.XML ->
|
||||
GeoFeed(
|
||||
Regex("<location\\s+([^>]+)/>")
|
||||
.findAll(payload)
|
||||
.map { xmlToRecord(it.groupValues[1]) }
|
||||
.toList()
|
||||
)
|
||||
}
|
||||
|
||||
private fun csvRecord(line: String): GeoLocation {
|
||||
val fields = splitCsv(line)
|
||||
require(fields.size >= 2) { "RFC 8805 record needs a prefix and country" }
|
||||
return GeoLocation(
|
||||
fields[0],
|
||||
fields[1].uppercase(),
|
||||
fields.getOrNull(2).blankToNull(),
|
||||
fields.getOrNull(3).blankToNull(),
|
||||
fields.getOrNull(4).blankToNull(),
|
||||
source = "rfc8805",
|
||||
)
|
||||
}
|
||||
|
||||
private fun jsonRecord(r: GeoLocation) =
|
||||
"{" +
|
||||
listOf(
|
||||
"prefix" to r.prefix,
|
||||
"countryCode" to r.countryCode,
|
||||
"region" to r.region,
|
||||
"city" to r.city,
|
||||
"postalCode" to r.postalCode,
|
||||
"latitude" to r.latitude,
|
||||
"longitude" to r.longitude,
|
||||
"source" to r.source,
|
||||
"observedAt" to r.observedAt.toString(),
|
||||
"confidence" to r.confidence,
|
||||
)
|
||||
.joinToString(",") { (k, v) ->
|
||||
"\"$k\":" +
|
||||
when (v) {
|
||||
null -> "null"
|
||||
is Number -> v.toString()
|
||||
else -> "\"${json(v.toString())}\""
|
||||
}
|
||||
} +
|
||||
"}"
|
||||
|
||||
private fun jsonToRecord(obj: String): GeoLocation {
|
||||
fun value(name: String) =
|
||||
Regex("\"$name\"\\s*:\\s*(null|\"((?:\\\\.|[^\"])*)\"|[-.0-9]+)").find(obj)?.let {
|
||||
if (it.groupValues[1] == "null") null
|
||||
else
|
||||
it.groupValues[2]
|
||||
.ifEmpty { it.groupValues[1] }
|
||||
.replace("\\\"", "\"")
|
||||
.replace("\\\\", "\\")
|
||||
}
|
||||
return GeoLocation(
|
||||
value("prefix") ?: error("JSON record missing prefix"),
|
||||
value("countryCode") ?: error("JSON record missing countryCode"),
|
||||
value("region"),
|
||||
value("city"),
|
||||
value("postalCode"),
|
||||
value("latitude")?.toDouble(),
|
||||
value("longitude")?.toDouble(),
|
||||
value("source") ?: "commonip",
|
||||
value("observedAt")?.let(Instant::parse) ?: Instant.now(),
|
||||
value("confidence")?.toDouble() ?: 0.5,
|
||||
)
|
||||
}
|
||||
|
||||
private fun xmlToRecord(attrs: String): GeoLocation {
|
||||
fun value(name: String) =
|
||||
Regex("$name=\"([^\"]*)\"").find(attrs)?.groupValues?.get(1)?.let {
|
||||
it.replace(""", "\"").replace("&", "&")
|
||||
}
|
||||
return GeoLocation(
|
||||
value("prefix") ?: error("XML record missing prefix"),
|
||||
value("country") ?: error("XML record missing country"),
|
||||
value("region"),
|
||||
value("city"),
|
||||
value("postalCode"),
|
||||
value("latitude")?.toDouble(),
|
||||
value("longitude")?.toDouble(),
|
||||
value("source") ?: "commonip",
|
||||
value("observedAt")?.let(Instant::parse) ?: Instant.now(),
|
||||
value("confidence")?.toDouble() ?: 0.5,
|
||||
)
|
||||
}
|
||||
|
||||
private fun jsonObjects(raw: String): List<String> =
|
||||
Regex("\\{(?:[^{}]|\\{[^{}]*})*}").findAll(raw).map { it.value }.toList()
|
||||
|
||||
private fun splitCsv(line: String): List<String> =
|
||||
Regex("(?:^|,)(\"(?:[^\"]|\"\")*\"|[^,]*)")
|
||||
.findAll(line)
|
||||
.map { it.groupValues[1].removeSurrounding("\"").replace("\"\"", "\"") }
|
||||
.toList()
|
||||
|
||||
private fun csv(value: String) =
|
||||
if (value.any { it == ',' || it == '"' || it == '\n' }) "\"${value.replace("\"", "\"\"")}\""
|
||||
else value
|
||||
|
||||
private fun json(value: String) = value.replace("\\", "\\\\").replace("\"", "\\\"")
|
||||
|
||||
private fun xml(value: String) = value.replace("&", "&").replace("\"", """)
|
||||
|
||||
private fun attr(name: String, value: Any?) =
|
||||
value?.let { " $name=\"${xml(it.toString())}\"" }.orEmpty()
|
||||
|
||||
private fun String?.blankToNull() = this?.takeIf { it.isNotBlank() }
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
package rip.crit.commonip.serialize
|
||||
|
||||
fun GeoFeed.serialize(format: GeoFeedFormat): String = GeoFeedCodec.encode(this, format)
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package rip.crit.commonip.serialize
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class GeoFeedCodecTest {
|
||||
@Test
|
||||
fun `RFC 8805 records round trip`() {
|
||||
val feed = GeoFeed(listOf(GeoLocation("192.0.2.0/24", "US", "PA", "Philadelphia", "19106")))
|
||||
val parsed =
|
||||
GeoFeedCodec.decode(GeoFeedCodec.encode(feed, GeoFeedFormat.CSV), GeoFeedFormat.CSV)
|
||||
assertEquals(feed.records.first().prefix, parsed.records.first().prefix)
|
||||
assertEquals("Philadelphia", parsed.records.first().city)
|
||||
}
|
||||
}
|
||||
3
packages/settings.gradle.kts
Normal file
3
packages/settings.gradle.kts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
|
||||
rootProject.name = "commonip-packages"
|
||||
include("decentraland", "serialize", "ml", "rawr")
|
||||
74
proprietary/build.gradle.kts
Normal file
74
proprietary/build.gradle.kts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import java.net.URI
|
||||
import javax.xml.parsers.DocumentBuilderFactory
|
||||
import org.w3c.dom.Element
|
||||
import org.w3c.dom.Node
|
||||
|
||||
plugins {
|
||||
id("buildlogic.kotlin-library-conventions")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api(project(":packages:serialize"))
|
||||
implementation(project(":packages:rawr"))
|
||||
implementation("commons-net:commons-net:3.11.1")
|
||||
}
|
||||
|
||||
tasks.register("whois") {
|
||||
group = "generation"
|
||||
description = "Fetches latest WHOIS server mapping and generates WhoisServers.kt in source."
|
||||
|
||||
val outputDirectory = file("src/main/kotlin/rip/crit/commonip/proprietary")
|
||||
outputs.file(outputDirectory.resolve("WhoisServers.kt"))
|
||||
|
||||
doLast {
|
||||
val xmlUrl =
|
||||
"https://raw.githubusercontent.com/whois-server-list/whois-server-list/refs/heads/master/whois-server-list.xml"
|
||||
|
||||
val inputStream = URI(xmlUrl).toURL().openStream()
|
||||
val dbFactory = DocumentBuilderFactory.newInstance()
|
||||
val dBuilder = dbFactory.newDocumentBuilder()
|
||||
val xmlDoc = dBuilder.parse(inputStream)
|
||||
xmlDoc.documentElement.normalize()
|
||||
|
||||
val serverMap = mutableMapOf<String, String>()
|
||||
val domainNodes = xmlDoc.getElementsByTagName("domain")
|
||||
|
||||
for (i in 0 until domainNodes.length) {
|
||||
val domainNode = domainNodes.item(i)
|
||||
if (domainNode.nodeType == Node.ELEMENT_NODE) {
|
||||
val domainElement = domainNode as Element
|
||||
val domainName = domainElement.getAttribute("name")
|
||||
|
||||
val whoisServerNodes = domainElement.getElementsByTagName("whoisServer")
|
||||
if (whoisServerNodes.length > 0) {
|
||||
val whoisServerElement = whoisServerNodes.item(0) as Element
|
||||
val host = whoisServerElement.getAttribute("host")
|
||||
if (host.isNotBlank() && domainName.isNotBlank()) {
|
||||
serverMap[domainName] = host
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val mapEntries = StringBuilder()
|
||||
serverMap.forEach { (tld, server) ->
|
||||
mapEntries.append(" \"$tld\" to \"$server\",\n")
|
||||
}
|
||||
|
||||
val outputFile = outputDirectory.resolve("WhoisServers.kt")
|
||||
outputFile.parentFile.mkdirs()
|
||||
outputFile.writeText(
|
||||
"""
|
||||
package rip.crit.commonip.proprietary
|
||||
|
||||
// Auto-generated file. Do not edit directly.
|
||||
object WhoisServers {
|
||||
val map: Map<String, String> = mapOf(
|
||||
$mapEntries )
|
||||
}
|
||||
"""
|
||||
.trimIndent()
|
||||
)
|
||||
println("Successfully generated ${outputFile.path} with ${serverMap.size} active TLD servers.")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import java.net.URI
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
|
||||
fun interface HttpTransport {
|
||||
fun get(url: String, headers: Map<String, String>): String
|
||||
}
|
||||
|
||||
object DefaultHttpTransport : HttpTransport {
|
||||
private val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
|
||||
|
||||
override fun get(url: String, headers: Map<String, String>): String {
|
||||
val request = HttpRequest.newBuilder(URI(url)).apply { headers.forEach(::header) }.GET().build()
|
||||
return client.send(request, HttpResponse.BodyHandlers.ofString()).body()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import rip.crit.commonip.rawr.types.Cidr
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||
import rip.crit.commonip.serialize.GeoLocation
|
||||
|
||||
class OpenGeoFeed(private val transport: HttpTransport = DefaultHttpTransport) {
|
||||
fun pull(url: String): GeoFeed =
|
||||
GeoFeedCodec.decode(transport.get(url, emptyMap()), GeoFeedFormat.CSV)
|
||||
|
||||
fun lookup(url: String, address: String): GeoLocation? =
|
||||
pull(url)
|
||||
.records
|
||||
.filter { Cidr.contains(it.prefix, address) }
|
||||
.maxByOrNull { Cidr.prefixLength(it.prefix) }
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
class Rdap(private val transport: HttpTransport = DefaultHttpTransport) {
|
||||
fun lookup(address: String): String =
|
||||
transport.get("https://rdap.org/ip/$address", mapOf("Accept" to "application/rdap+json"))
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import java.time.Duration
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.Semaphore
|
||||
|
||||
class RequestGovernor(
|
||||
private val minimumInterval: Duration = Duration.ofMillis(500),
|
||||
parallelism: Int = 2,
|
||||
) {
|
||||
private val permits = Semaphore(parallelism)
|
||||
private val lastRequest = ConcurrentHashMap<String, Long>()
|
||||
|
||||
fun <T> execute(provider: String, action: () -> T): T {
|
||||
permits.acquire()
|
||||
try {
|
||||
synchronized(lastRequest) {
|
||||
val wait =
|
||||
minimumInterval.toMillis() - (System.currentTimeMillis() - (lastRequest[provider] ?: 0))
|
||||
if (wait > 0) Thread.sleep(wait)
|
||||
lastRequest[provider] = System.currentTimeMillis()
|
||||
}
|
||||
return action()
|
||||
} finally {
|
||||
permits.release()
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> chunks(values: Collection<T>, size: Int = 100): Sequence<List<T>> =
|
||||
values.asSequence().chunked(size)
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import rip.crit.commonip.serialize.GeoFeed
|
||||
import rip.crit.commonip.serialize.GeoFeedCodec
|
||||
import rip.crit.commonip.serialize.GeoFeedFormat
|
||||
|
||||
class StarlinkGeoFeed(private val transport: HttpTransport = DefaultHttpTransport) {
|
||||
companion object {
|
||||
const val URL = "https://geoip.starlinkisp.net/feed.csv"
|
||||
}
|
||||
|
||||
fun pull(): GeoFeed = GeoFeedCodec.decode(transport.get(URL, emptyMap()), GeoFeedFormat.CSV)
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import org.apache.commons.net.whois.WhoisClient
|
||||
|
||||
class Whois(private val servers: Map<String, String> = WhoisServers.map) {
|
||||
fun lookup(query: String): String {
|
||||
val server =
|
||||
servers[query.substringAfterLast('.', "").lowercase()] ?: servers.getValue("default")
|
||||
val client = WhoisClient()
|
||||
return try {
|
||||
client.connect(server)
|
||||
client.query("$query\r\n")
|
||||
} finally {
|
||||
if (client.isConnected) client.disconnect()
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,13 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable
|
||||
|
||||
class LiveProviderTest {
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "COMMONIP_LIVE_TESTS", matches = "true")
|
||||
fun `starlink live feed has records in isolated opt in test`() {
|
||||
assertTrue(StarlinkGeoFeed().pull().records.isNotEmpty())
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package rip.crit.commonip.proprietary
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class ProviderTest {
|
||||
@Test
|
||||
fun `open geofeed uses injected fixture transport`() {
|
||||
val feed =
|
||||
OpenGeoFeed(HttpTransport { _, _ -> "203.0.113.0/24,US,CA,Los Angeles,90001" })
|
||||
.pull("https://fixture.invalid/feed.csv")
|
||||
assertEquals("Los Angeles", feed.records.single().city)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `starlink endpoint is opt in provider`() {
|
||||
assertEquals("https://geoip.starlinkisp.net/feed.csv", StarlinkGeoFeed.URL)
|
||||
}
|
||||
}
|
||||
12
scripts/commonip.service
Normal file
12
scripts/commonip.service
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
[Unit]
|
||||
Description=CommonIP node
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
User=commonip
|
||||
WorkingDirectory=/opt/commonip
|
||||
ExecStart=/opt/commonip/commonip
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
14
scripts/deploy-ssh.sh
Executable file
14
scripts/deploy-ssh.sh
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
deploy_host="${1:?host is required}"
|
||||
deploy_user="${2:?user is required}"
|
||||
deploy_path="${COMMONIP_REMOTE_PATH:-/opt/commonip}"
|
||||
|
||||
./gradlew :app:nativeImage
|
||||
|
||||
ssh "${deploy_user}@${deploy_host}" "sudo useradd --system --create-home --shell /usr/sbin/nologin commonip 2>/dev/null || true; sudo install -d -o commonip -g commonip ${deploy_path}"
|
||||
scp app/build/native/commonip "${deploy_user}@${deploy_host}:/tmp/commonip"
|
||||
ssh "${deploy_user}@${deploy_host}" "sudo install -o commonip -g commonip -m 0755 /tmp/commonip ${deploy_path}/commonip"
|
||||
scp scripts/commonip.service "${deploy_user}@${deploy_host}:/tmp/commonip.service"
|
||||
ssh "${deploy_user}@${deploy_host}" "sudo install -m 0644 /tmp/commonip.service /etc/systemd/system/commonip.service; sudo systemctl daemon-reload; sudo systemctl enable --now commonip"
|
||||
13
settings.gradle.kts
Normal file
13
settings.gradle.kts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
pluginManagement {
|
||||
includeBuild("build-logic")
|
||||
}
|
||||
|
||||
plugins {
|
||||
id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0"
|
||||
}
|
||||
|
||||
rootProject.name = "commonip"
|
||||
|
||||
include("app", "proprietary", "packages")
|
||||
include(":packages:serialize", ":packages:rawr", ":packages:ml", ":packages:decentraland")
|
||||
include(":packages:decentraland:kademlia", ":packages:decentraland:git", ":packages:decentraland:chksum")
|
||||
Loading…
Reference in a new issue