Initial commit.

This commit is contained in:
Hellings 2026-09-23 23:39:11 -04:00
commit 89a7c8eaef
90 changed files with 3920 additions and 0 deletions

12
.gitattributes vendored Normal file
View file

@ -0,0 +1,12 @@
#
# https://help.github.com/articles/dealing-with-line-endings/
#
# Linux start script should use lf
/gradlew text eol=lf
# These are Windows script files and should use crlf
*.bat text eol=crlf
# Binary files should be left untouched
*.jar binary

12
.gitignore vendored Normal file
View file

@ -0,0 +1,12 @@
# Ignore Gradle project-specific cache directory
.gradle
# Ignore Gradle build output directory
build
# Ignore Kotlin plugin data
.kotlin
# Node credentials and deployment-specific endpoints
node.yaml
deploy/*.yaml

16
Justfile Normal file
View file

@ -0,0 +1,16 @@
set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
build:
./gradlew build
fmt:
./gradlew spotlessApply
check:
./gradlew spotlessCheck build
native:
./gradlew nativeImage
deploy host user:
scripts/deploy-ssh.sh {{host}} {{user}}

12
README.md Normal file
View file

@ -0,0 +1,12 @@
# CommonIP
CommonIP, not to be confused with [CommonApp](https://commonapp.org), is a decentralized geolocation network that specifically avoids proprietary geolocation services whenever possible. It's written in the ever-so-portable Kotlin. Use it from here, use it from there, use it from anywhere!
# donate
Please donate to me to keep this project and its (centralized) [geofeeds](https://ai.crit.rip/automatedintelligence/geofeeds) alive.
- `eth` 0xF30C281Aa2CD94618D902AE5bb9101E9F5385fE3
- `sol` 99zJPASUz9PKLFaJWrxHCEtLZxeEePYpgZ74Ly9YNUHy
- `paypal` +1 (267) 806-2744 - Stephen Hellings
- `zelle` +1 (267) 806-2744 - STEPHEN HELLINGS

32
app/build.gradle.kts Normal file
View file

@ -0,0 +1,32 @@
import buildlogic.GraalNativeImageTask
plugins {
id("buildlogic.kotlin-application-conventions")
}
dependencies {
implementation("org.apache.commons:commons-text")
implementation(project(":packages:serialize"))
implementation(project(":packages:rawr"))
implementation(project(":proprietary"))
implementation(project(":packages:ml"))
implementation(project(":packages:decentraland"))
}
application {
mainClass = "rip.crit.commonip.app.MainKtKt"
}
tasks.named<JavaExec>("run") {
standardInput = System.`in`
}
val nativeImage =
tasks.register<GraalNativeImageTask>("nativeImage") {
group = "distribution"
description = "Builds a GraalVM native executable with the active Gradle toolchain."
dependsOn(tasks.named("classes"))
classpath.set(sourceSets.main.get().runtimeClasspath.files.map { it.absolutePath })
mainClass.set("rip.crit.commonip.app.MainKtKt")
output.set(layout.buildDirectory.file("native/commonip"))
}

View file

@ -0,0 +1,93 @@
package rip.crit.commonip.app
import java.net.InetSocketAddress
import java.util.concurrent.ConcurrentHashMap
import rip.crit.commonip.decentraland.git.RepositoryTrust
import rip.crit.commonip.decentraland.kademlia.KademliaNode
import rip.crit.commonip.decentraland.kademlia.NodeRegistry
import rip.crit.commonip.ml.GeoLocation as MlGeoLocation
import rip.crit.commonip.ml.LocationEstimator
import rip.crit.commonip.proprietary.OpenGeoFeed
import rip.crit.commonip.proprietary.StarlinkGeoFeed
import rip.crit.commonip.rawr.providers.RirGeoFeedLoader
import rip.crit.commonip.rawr.providers.RirMetadataField
import rip.crit.commonip.rawr.types.Cidr
import rip.crit.commonip.serialize.GeoLocation
class ClusterGeolocation(private val registry: NodeRegistry, bind: InetSocketAddress) :
AutoCloseable {
private val feeds = ConcurrentHashMap<String, RepositoryTrust>()
private val local = ConcurrentHashMap<String, GeoLocation>()
private val dht = KademliaNode(bind, registry)
fun start(seeds: List<InetSocketAddress> = emptyList()) {
dht.start()
seeds.forEach(dht::bootstrap)
}
fun addFeed(url: String, trust: RepositoryTrust = RepositoryTrust.COMMUNITY) {
feeds[url] = trust
}
fun addStarlinkFeed() {
feeds[StarlinkGeoFeed.URL] = RepositoryTrust.TAINTED
}
fun pullRirRegistryObject(
source: String,
registryObject: String,
fields: Set<RirMetadataField>,
trust: RepositoryTrust = RepositoryTrust.COMMUNITY,
) {
RirGeoFeedLoader(source, fields).pull(registryObject).forEach { feed ->
feed.records.forEach { record ->
local[record.prefix] = record
dht.store(
"${if (trust == RepositoryTrust.TAINTED) "commonip/tainted-full" else "commonip/community"}:${record.prefix}",
record.prefix,
)
}
registry.publish(
"local",
"rir geofeed pulled",
"$source yielded ${feed.records.size} records",
)
}
}
fun pullAll() {
feeds.forEach { (url, trust) ->
runCatching { OpenGeoFeed().pull(url).records }
.onSuccess { records ->
records.forEach { record ->
local[record.prefix] = record
dht.store(
"${if (trust == RepositoryTrust.TAINTED) "commonip/tainted-full" else "commonip/community"}:${record.prefix}",
record.prefix,
)
}
registry.publish(
"local",
"${trust.name.lowercase()} feed pulled",
"$url yielded ${records.size} records",
)
}
.onFailure { registry.publish("local", "feed pull failed", "$url ${it.message}") }
}
}
fun lookup(address: String): GeoLocation? =
local.values
.filter { Cidr.contains(it.prefix, address) }
.maxByOrNull { Cidr.prefixLength(it.prefix) }
fun estimate(address: String) =
LocationEstimator()
.estimate(
local.values
.filter { Cidr.contains(it.prefix, address) }
.map { MlGeoLocation(it.latitude, it.longitude, it.confidence) }
)
override fun close() = dht.close()
}

View file

@ -0,0 +1,116 @@
package rip.crit.commonip.app
import java.net.InetSocketAddress
import java.nio.file.Path
import java.nio.file.Files
import rip.crit.commonip.decentraland.git.NodeYamlReader
import rip.crit.commonip.decentraland.SshDashboardService
import rip.crit.commonip.decentraland.kademlia.NetworkNode
import rip.crit.commonip.decentraland.kademlia.NodeRegistry
import rip.crit.commonip.decentraland.kademlia.NodeState
fun main() = CommonIpNode().run()
private class CommonIpNode {
private val registry = NodeRegistry()
private val cluster =
ClusterGeolocation(
registry,
InetSocketAddress(System.getenv("COMMONIP_DHT_PORT")?.toIntOrNull() ?: 7400),
)
fun run() {
val configPath = Path.of(System.getenv("COMMONIP_NODE_CONFIG") ?: "/etc/commonip/node.yaml")
val configuredBootstrap = if (Files.isRegularFile(configPath)) NodeYamlReader.parse(Files.readString(configPath)).bootstrap else emptyList()
cluster.start((configuredBootstrap + System.getenv("COMMONIP_BOOTSTRAP")?.split(',').orEmpty()).mapNotNull(::parseEndpoint).distinct())
System.getenv("COMMONIP_GEOFEEDS")
?.split(',')
?.filter(String::isNotBlank)
?.forEach(cluster::addFeed)
if (System.getenv("COMMONIP_ENABLE_TAINTED_STARLINK") == "true") cluster.addStarlinkFeed()
cluster.pullAll()
val sshService =
System.getenv("COMMONIP_SSH_PASSWD")?.let { password ->
SshDashboardService(
System.getenv("COMMONIP_SSH_PORT")?.toIntOrNull() ?: 2222,
Path.of(System.getenv("COMMONIP_SSH_HOST_KEY") ?: "/opt/commonip/hostkey.ser"),
System.getenv("COMMONIP_SSH_USER") ?: "commonip",
password.toCharArray(),
) {
registry.snapshot().joinToString("\n") { "${it.id} ${it.state}" }
}
.also { it.start() }
}
if (System.console() == null) {
while (true) {
Thread.sleep(60_000)
cluster.pullAll()
}
}
sshService?.close()
while (true) {
render()
print("commonip/network> ")
val command = readlnOrNull()?.trim()?.lowercase() ?: return
when (command) {
"q",
"quit",
"exit" -> return
"r",
"refresh" -> cluster.pullAll()
"l",
"lookup" -> lookup()
"e",
"estimate" -> estimate()
else -> println("Commands are r refresh, l lookup, e estimate, q quit.")
}
}
}
private fun parseEndpoint(value: String): InetSocketAddress? =
value.trim().split(':', limit = 2).let { parts ->
parts.getOrNull(1)?.toIntOrNull()?.let { InetSocketAddress(parts[0], it) }
}
private fun render() {
print("\u001b[H\u001b[2J")
val nodes = registry.snapshot()
println()
nodes.chunked(8).forEach { row ->
println(row.joinToString(" ") { node -> "${status(node)} ${node.id}" })
}
println()
registry.feed().forEach { event ->
println(
"${event.at.toString().substring(11, 19)} ${event.nodeId} ${event.action} ${event.detail}"
)
}
}
private fun status(node: NetworkNode): String =
when (node.state) {
NodeState.ONLINE -> "\u001b[32m●\u001b[0m"
NodeState.DEGRADED -> "\u001b[33m●\u001b[0m"
NodeState.OFFLINE -> "\u001b[90m●\u001b[0m"
}
private fun lookup() {
print("IPv4 address> ")
val address = readlnOrNull()?.trim().orEmpty()
val match = cluster.lookup(address)
registry.publish("local", "lookup", match?.prefix ?: "no matching CIDR for $address")
}
private fun estimate() {
print("CIDR or IPv4 address> ")
val address = readlnOrNull()?.trim().orEmpty()
val result = cluster.estimate(address)
registry.publish(
"local",
"estimate",
result?.let {
"%.4f, %.4f confidence %.2f".format(it.latitude, it.longitude, it.confidence)
} ?: "no coordinate observations",
)
}
}

View file

@ -0,0 +1,18 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
`kotlin-dsl`
}
repositories {
gradlePluginPortal()
}
dependencies {
implementation(libs.kotlin.gradle.plugin)
implementation("com.diffplug.spotless:com.diffplug.spotless.gradle.plugin:8.10.2")
}

View file

@ -0,0 +1,14 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This settings file is used to specify which projects to include in your build-logic build.
* This project uses @Incubating APIs which are subject to change.
*/
dependencyResolutionManagement {
versionCatalogs {
create("libs", { from(files("../gradle/libs.versions.toml")) })
}
}
rootProject.name = "build-logic"

View file

@ -0,0 +1,11 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
id("buildlogic.kotlin-common-conventions")
application
}

View file

@ -0,0 +1,54 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
id("org.jetbrains.kotlin.jvm")
id("com.diffplug.spotless")
}
repositories {
mavenCentral()
}
dependencies {
constraints {
implementation("org.apache.commons:commons-text:1.14.0")
}
}
testing {
suites {
val test = named<JvmTestSuite>("test") {
useJUnitJupiter("6.0.1")
}
}
}
tasks.named("check") {
dependsOn("spotlessCheck")
}
java {
toolchain {
languageVersion = JavaLanguageVersion.of(25)
vendor = JvmVendorSpec.GRAAL_VM
}
}
spotless {
kotlin {
target("src/**/*.kt")
ktfmt()
trimTrailingWhitespace()
endWithNewline()
}
kotlinGradle {
target("*.gradle.kts")
ktfmt()
trimTrailingWhitespace()
endWithNewline()
}
}

View file

@ -0,0 +1,17 @@
plugins {
id("com.diffplug.spotless")
}
spotless {
java {
target("src/**/*.java")
importOrder()
removeUnusedImports()
palantirJavaFormat()
}
kotlin {
target("src/**/*.kt")
ktlint()
}
}

View file

@ -0,0 +1,11 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
id("buildlogic.kotlin-common-conventions")
`java-library`
}

View file

@ -0,0 +1,30 @@
package buildlogic
import javax.inject.Inject
import org.gradle.api.DefaultTask
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.ListProperty
import org.gradle.api.tasks.Classpath
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import org.gradle.jvm.toolchain.JavaLanguageVersion
import org.gradle.jvm.toolchain.JavaToolchainService
import org.gradle.jvm.toolchain.JvmVendorSpec
import org.gradle.process.ExecOperations
abstract class GraalNativeImageTask @Inject constructor(private val toolchains: JavaToolchainService, private val exec: ExecOperations) : DefaultTask() {
@get:Classpath abstract val classpath: ListProperty<String>
@get:OutputFile abstract val output: RegularFileProperty
abstract val mainClass: org.gradle.api.provider.Property<String>
@TaskAction fun build() {
val launcher = toolchains.launcherFor {
languageVersion.set(JavaLanguageVersion.of(25))
vendor.set(JvmVendorSpec.GRAAL_VM)
}.get()
val nativeImage = launcher.metadata.installationPath.file("bin/native-image").asFile
check(nativeImage.canExecute()) { "The configured Java 25 toolchain must be GraalVM with native-image installed" }
output.get().asFile.parentFile.mkdirs()
exec.exec { commandLine(nativeImage.absolutePath, "-cp", classpath.get().joinToString(java.io.File.pathSeparator), mainClass.get(), output.get().asFile.absolutePath) }
}
}

7
gradle.properties Normal file
View file

@ -0,0 +1,7 @@
# This file was generated by the Gradle 'init' task.
# https://docs.gradle.org/current/userguide/build_environment.html#sec:gradle_configuration_properties
org.gradle.configuration-cache=true
org.gradle.parallel=true
org.gradle.caching=true
org.gradle.java.installations.auto-download=true

View file

@ -0,0 +1,8 @@
# This file was generated by the Gradle 'init' task.
# https://docs.gradle.org/current/userguide/version_catalogs.html#sec::toml-dependencies-format
[versions]
kotlin-gradle-plugin = "2.4.0"
[libraries]
kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin-gradle-plugin" }

BIN
gradle/wrapper/gradle-wrapper.jar vendored Normal file

Binary file not shown.

View file

@ -0,0 +1,9 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.1-bin.zip
networkTimeout=10000
retries=0
retryBackOffMs=500
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

248
gradlew vendored Executable file
View file

@ -0,0 +1,248 @@
#!/bin/sh
#
# Copyright © 2015 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#
##############################################################################
#
# gradlew start up script for POSIX generated by Gradle.
#
# Important for running:
#
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
# noncompliant, but you have some other compliant shell such as ksh or
# bash, then to run this script, type that shell name before the whole
# command line, like:
#
# ksh gradlew
#
# Busybox and similar reduced shells will NOT work, because this script
# requires all of these POSIX shell features:
# * functions;
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
# * compound commands having a testable exit status, especially «case»;
# * various built-in commands including «command», «set», and «ulimit».
#
# Important for patching:
#
# (2) This script targets any POSIX shell, so it avoids extensions provided
# by Bash, Ksh, etc; in particular arrays are avoided.
#
# The "traditional" practice of packing multiple parameters into a
# space-separated string is a well documented source of bugs and security
# problems, so this is (mostly) avoided, by progressively accumulating
# options in "$@", and eventually passing that to Java.
#
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
# see the in-line comments for details.
#
# There are tweaks for specific operating systems such as AIX, CygWin,
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
#
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
app_path=$0
# Need this for daisy-chained symlinks.
while
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
[ -h "$app_path" ]
do
ls=$( ls -ld "$app_path" )
link=${ls#*' -> '}
case $link in #(
/*) app_path=$link ;; #(
*) app_path=$APP_HOME$link ;;
esac
done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
warn () {
echo "$*"
} >&2
die () {
echo
echo "$*"
echo
exit 1
} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "$( uname )" in #(
CYGWIN* ) cygwin=true ;; #(
Darwin* ) darwin=true ;; #(
MSYS* | MINGW* ) msys=true ;; #(
NONSTOP* ) nonstop=true ;;
esac
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD=$JAVA_HOME/jre/sh/java
else
JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD=java
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
case $MAX_FD in #(
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
fi
# Collect all arguments for the java command, stacking in reverse order:
# * args from the command line
# * the main class name
# * -classpath
# * -D...appname settings
# * --module-path (only if needed)
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
JAVACMD=$( cygpath --unix "$JAVACMD" )
# Now convert the arguments - kludge to limit ourselves to /bin/sh
for arg do
if
case $arg in #(
-*) false ;; # don't mess with options #(
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
[ -e "$t" ] ;; #(
*) false ;;
esac
then
arg=$( cygpath --path --ignore --mixed "$arg" )
fi
# Roll the args list around exactly as many times as the number of
# args, so each arg winds up back in the position where it started, but
# possibly modified.
#
# NB: a `for` loop captures its iteration list before it begins, so
# changing the positional parameters here affects neither the number of
# iterations, nor the values presented in `arg`.
shift # remove old arg
set -- "$@" "$arg" # push replacement arg
done
fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@"
# Stop when "xargs" is not available.
if ! command -v xargs >/dev/null 2>&1
then
die "xargs is not available"
fi
# Use "xargs" to parse quoted args.
#
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
#
# In Bash we could simply go:
#
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
# set -- "${ARGS[@]}" "$@"
#
# but POSIX shell has neither arrays nor command substitution, so instead we
# post-process each arg (as a line of input to sed) to backslash-escape any
# character that might be a shell metacharacter, then use eval to reverse
# that process (while maintaining the separation between arguments), and wrap
# the whole thing up as a single "set" statement.
#
# This will of course break if any of these variables contains a newline or
# an unmatched quote.
#
eval "set -- $(
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
xargs -n1 |
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
tr '\n' ' '
)" '"$@"'
exec "$JAVACMD" "$@"

82
gradlew.bat vendored Normal file
View file

@ -0,0 +1,82 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@rem SPDX-License-Identifier: Apache-2.0
@rem
@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
@rem
@rem gradlew startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables, and ensure extensions are enabled
setlocal EnableExtensions
set DIRNAME=%~dp0
if "%DIRNAME%"=="" set DIRNAME=.
@rem This is normally unused
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if %ERRORLEVEL% equ 0 goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
"%COMSPEC%" /c exit 1
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
"%COMSPEC%" /c exit 1
:execute
@rem Setup the command line
@rem Execute gradlew
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
@rem which allows us to clear the local environment before executing the java command
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
:exitWithErrorLevel
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
"%COMSPEC%" /c exit %ERRORLEVEL%

7
node.yaml.example Normal file
View file

@ -0,0 +1,7 @@
remote: https://ai.crit.rip/automatedintelligence/geofeeds.git
branch: main
lfs: true
tainted: false
passwd: "set-through-secret-manager"
bootstrap:
- "peer.example.net:7400"

View file

@ -0,0 +1 @@
plugins { base }

View file

@ -0,0 +1,10 @@
plugins {
id("buildlogic.kotlin-library-conventions")
}
dependencies {
api(project(":packages:serialize"))
api(project(":packages:decentraland:kademlia"))
api(project(":packages:decentraland:git"))
implementation("org.apache.sshd:sshd-core:2.15.0")
}

View file

@ -0,0 +1 @@
plugins { id("buildlogic.kotlin-library-conventions") }

View file

@ -0,0 +1,10 @@
package rip.crit.commonip.decentraland.chksum
import java.security.MessageDigest
object ContentHash {
fun sha256(value: String): String =
MessageDigest.getInstance("SHA-256").digest(value.toByteArray()).joinToString("") {
"%02x".format(it)
}
}

View file

@ -0,0 +1,3 @@
plugins { id("buildlogic.kotlin-library-conventions") }
dependencies { api(project(":packages:serialize")) }

View file

@ -0,0 +1,24 @@
package rip.crit.commonip.decentraland.git
import rip.crit.commonip.serialize.GeoLocation
enum class Taint {
CLEAN,
TAINTED,
}
data class ProvenancedObservation(
val location: GeoLocation,
val provider: String,
val taint: Taint,
val evidence: Set<String>,
)
object PromotionPolicy {
fun canPublishToCommunity(value: ProvenancedObservation): Boolean =
value.taint == Taint.CLEAN && value.evidence.size >= 2
fun destination(value: ProvenancedObservation): CommonIpRepository =
if (canPublishToCommunity(value)) CommonIpRepositories.community
else CommonIpRepositories.taintedFull
}

View file

@ -0,0 +1,67 @@
package rip.crit.commonip.decentraland.git
import java.nio.file.Path
enum class RepositoryTrust {
COMMUNITY,
TAINTED,
}
data class CommonIpRepository(
val name: String,
val trust: RepositoryTrust,
val graph: RevisionGraph = RevisionGraph(),
)
object CommonIpRepositories {
val community = CommonIpRepository("commonip/community", RepositoryTrust.COMMUNITY)
val taintedFull = CommonIpRepository("commonip/tainted-full", RepositoryTrust.TAINTED)
}
data class NodeYaml(
val remote: String,
val branch: String = "main",
val lfs: Boolean = true,
val tainted: Boolean = false,
val passwd: String? = null,
val bootstrap: List<String> = emptyList(),
)
object NodeYamlReader {
fun parse(text: String): NodeYaml {
val values =
text
.lineSequence()
.map(String::trim)
.filter { ':' in it && !it.startsWith('#') }
.associate {
it.substringBefore(':').trim() to it.substringAfter(':').trim().trim('"', '\'')
}
val bootstrap = text.lineSequence().map(String::trim).filter { it.startsWith("-") }.map { it.removePrefix("-").trim().trim('"', '\'') }.toList()
return NodeYaml(
values["remote"] ?: error("node.yaml requires remote"),
values["branch"] ?: "main",
values["lfs"]?.toBooleanStrictOrNull() ?: true,
values["tainted"]?.toBooleanStrictOrNull() ?: false,
values["passwd"],
bootstrap,
)
}
}
class GitLfsSync(private val repository: Path) {
fun sync(config: NodeYaml): Boolean =
runCatching {
fun command(vararg args: String) =
ProcessBuilder(*args)
.directory(repository.toFile())
.inheritIO()
.start()
.waitFor() == 0
command("git", "remote", "get-url", "origin") ||
command("git", "remote", "add", "origin", config.remote)
command("git", "fetch", "origin", config.branch) &&
(!config.lfs || command("git", "lfs", "pull", "origin", config.branch))
}
.getOrDefault(false)
}

View file

@ -0,0 +1,99 @@
package rip.crit.commonip.decentraland.git
import java.time.Instant
import java.util.UUID
import rip.crit.commonip.serialize.GeoLocation
data class GeoCommit(
val id: String = UUID.randomUUID().toString(),
val parents: Set<String>,
val author: String,
val observation: GeoLocation,
val createdAt: Instant = Instant.now(),
)
enum class MergeState {
FAST_FORWARD,
MERGED,
CONFLICT,
}
data class MergeResult(
val state: MergeState,
val commit: GeoCommit?,
val competing: List<GeoCommit>,
val message: String,
)
class RevisionGraph {
private val commits = linkedMapOf<String, GeoCommit>()
fun import(commit: GeoCommit): Boolean {
if (commit.parents.any { it !in commits }) return false
commits.putIfAbsent(commit.id, commit)
return true
}
fun commit(author: String, observation: GeoLocation, parents: Set<String> = heads()): GeoCommit =
GeoCommit(parents = parents, author = author, observation = observation).also {
commits[it.id] = it
}
fun heads(): Set<String> = commits.keys - commits.values.flatMap { it.parents }.toSet()
fun merge(left: String, right: String, author: String): MergeResult {
val a =
commits[left]
?: return MergeResult(
MergeState.CONFLICT,
null,
emptyList(),
"left revision is unknown",
)
val b =
commits[right]
?: return MergeResult(
MergeState.CONFLICT,
null,
emptyList(),
"right revision is unknown",
)
if (isAncestor(a.id, b.id))
return MergeResult(MergeState.FAST_FORWARD, b, emptyList(), "left fast-forwards to right")
if (isAncestor(b.id, a.id))
return MergeResult(MergeState.FAST_FORWARD, a, emptyList(), "right fast-forwards to left")
if (!compatible(a.observation, b.observation))
return MergeResult(
MergeState.CONFLICT,
null,
listOf(a, b),
"country or coordinates disagree, proof-of-location required",
)
val winner = listOf(a, b).maxBy { it.observation.confidence }
val merged =
GeoCommit(
parents = setOf(a.id, b.id),
author = author,
observation =
winner.observation.copy(
source = "merge:$author",
confidence = (a.observation.confidence + b.observation.confidence) / 2,
),
)
commits[merged.id] = merged
return MergeResult(MergeState.MERGED, merged, emptyList(), "two verified revisions merged")
}
private fun isAncestor(ancestor: String, descendant: String): Boolean =
commits[descendant]?.parents?.let { parents ->
ancestor in parents || parents.any { isAncestor(ancestor, it) }
} ?: false
private fun compatible(a: GeoLocation, b: GeoLocation): Boolean {
val aLat = a.latitude
val bLat = b.latitude
return a.prefix == b.prefix &&
a.countryCode == b.countryCode &&
(aLat == null || bLat == null || kotlin.math.abs(aLat - bLat) < 1.0)
}
}

View file

@ -0,0 +1,72 @@
package rip.crit.commonip.decentraland.git
import java.nio.charset.StandardCharsets
import java.security.PrivateKey
import java.security.PublicKey
import java.security.Signature
import java.time.Instant
import java.util.Base64
data class SignedGeoCommit(val revision: GeoCommit, val signature: String) {
fun payload(): ByteArray =
listOf(
revision.id,
revision.parents.sorted().joinToString(","),
revision.author,
revision.observation.prefix,
revision.observation.countryCode,
revision.observation.latitude,
revision.observation.longitude,
revision.createdAt,
)
.joinToString("|")
.toByteArray(StandardCharsets.UTF_8)
}
object CommitSigner {
fun sign(revision: GeoCommit, key: PrivateKey): SignedGeoCommit {
val unsigned = SignedGeoCommit(revision, "")
val signature =
Signature.getInstance("Ed25519")
.apply {
initSign(key)
update(unsigned.payload())
}
.sign()
return unsigned.copy(signature = Base64.getEncoder().encodeToString(signature))
}
fun verify(commit: SignedGeoCommit, key: PublicKey): Boolean =
runCatching {
Signature.getInstance("Ed25519")
.apply {
initVerify(key)
update(commit.payload())
}
.verify(Base64.getDecoder().decode(commit.signature))
}
.getOrDefault(false)
}
data class ProofOfLocation(
val revisionId: String,
val observer: String,
val observedAt: Instant,
val reachable: Boolean,
val latencyMs: Long?,
val signature: String,
)
class SignedCommitLedger(
private val graph: RevisionGraph,
private val keys: Map<String, PublicKey>,
) {
fun accept(commit: SignedGeoCommit): Boolean =
(keys[commit.revision.author]?.let { CommitSigner.verify(commit, it) } ?: false) &&
graph.import(commit.revision)
fun resolve(left: SignedGeoCommit, right: SignedGeoCommit, author: String): MergeResult {
require(accept(left) && accept(right)) { "all merge inputs must have valid signatures" }
return graph.merge(left.revision.id, right.revision.id, author)
}
}

View file

@ -0,0 +1,32 @@
package rip.crit.commonip.decentraland.git
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import rip.crit.commonip.serialize.GeoLocation
class RepositoryTest {
@Test
fun `node yaml retains trust and password fields`() {
val config = NodeYamlReader.parse("remote: ssh://example/repo\ntainted: true\npasswd: secret\nbootstrap:\n - peer.example:7400")
assertTrue(config.tainted)
assertEquals("secret", config.passwd)
assertEquals(listOf("peer.example:7400"), config.bootstrap)
}
@Test
fun `community promotion needs clean independent evidence`() {
val location = GeoLocation("192.0.2.0/24", "US")
assertFalse(
PromotionPolicy.canPublishToCommunity(
ProvenancedObservation(location, "test", Taint.CLEAN, setOf("one"))
)
)
assertTrue(
PromotionPolicy.canPublishToCommunity(
ProvenancedObservation(location, "test", Taint.CLEAN, setOf("one", "two"))
)
)
}
}

View file

@ -0,0 +1,7 @@
plugins { id("buildlogic.kotlin-library-conventions") }
dependencies {
api(project(":packages:serialize"))
implementation("io.ep2p:kademlia-api:5.1.2-RELEASE")
implementation("io.ep2p:kademlia-netty:0.3.2-RELEASE")
}

View file

@ -0,0 +1,111 @@
package rip.crit.commonip.decentraland.kademlia
import java.net.DatagramPacket
import java.net.DatagramSocket
import java.net.InetSocketAddress
import java.security.MessageDigest
import java.util.concurrent.ConcurrentHashMap
import kotlin.concurrent.thread
data class KademliaPeer(val id: String, val address: InetSocketAddress)
class KademliaNode(private val bind: InetSocketAddress, private val registry: NodeRegistry) :
AutoCloseable {
val id = sha256("${bind.hostString}:${bind.port}")
private val namer = NodeNamer()
private val peers = ConcurrentHashMap<String, KademliaPeer>()
private val values = ConcurrentHashMap<String, String>()
private val socket = DatagramSocket(bind)
private val lastOutbound = ConcurrentHashMap<String, Long>()
@Volatile private var running = true
fun start() =
thread(name = "commonip-kademlia-$id", isDaemon = true) {
registry.register(
NetworkNode(
namer.name(id),
socket.localSocketAddress.toString(),
NodeState.ONLINE,
0,
values.size,
)
)
val bytes = ByteArray(8192)
while (running) runCatching {
val packet = DatagramPacket(bytes, bytes.size)
socket.receive(packet)
handle(packet)
}
}
fun bootstrap(seed: InetSocketAddress) {
if (peers.values.none { it.address == seed }) send(seed, "PING|$id|${bind.hostString}|${socket.localPort}")
}
fun store(key: String, value: String) {
values[key] = value
closest(key).forEach { send(it.address, "STORE|$id|$key|${value.encodeBase64()}") }
}
fun findValue(key: String): String? = values[key]
fun closest(key: String, count: Int = 20): List<KademliaPeer> =
peers.values.sortedBy { xorDistance(it.id, key) }.take(count)
private fun handle(packet: DatagramPacket) {
val data =
packet.data
.copyOfRange(packet.offset, packet.offset + packet.length)
.decodeToString()
.split('|', limit = 5)
val remote = InetSocketAddress(packet.address, packet.port)
when (data.firstOrNull()) {
"PING" -> {
peers[data[1]] = KademliaPeer(data[1], InetSocketAddress(data[2], data[3].toInt()))
val name = namer.name(data[1])
registry.register(NetworkNode(name, remote.toString(), NodeState.ONLINE, null, 0))
registry.publish(name, "kademlia ping", remote.toString())
send(remote, "PONG|$id")
}
"PONG" -> {
peers[data[1]] = KademliaPeer(data[1], remote)
val name = namer.name(data[1])
registry.register(NetworkNode(name, remote.toString(), NodeState.ONLINE, null, 0))
registry.publish(name, "kademlia bootstrap confirmed", remote.toString())
}
"STORE" -> values[data[2]] = data[3].decodeBase64()
"FIND_VALUE" ->
values[data[2]]?.let { send(remote, "VALUE|$id|${data[2]}|${it.encodeBase64()}") }
"FIND_NODE" ->
send(
remote,
"NODES|$id|${closest(data[2]).joinToString(";") { "${it.id}@${it.address.hostString}:${it.address.port}" }}",
)
}
}
private fun send(target: InetSocketAddress, message: String) {
val key = "${target.hostString}:${target.port}:${message.substringBefore('|')}"
val previous = lastOutbound.put(key, System.currentTimeMillis()) ?: 0
if (System.currentTimeMillis() - previous < 500) return
socket.send(DatagramPacket(message.encodeToByteArray(), message.length, target))
}
override fun close() {
running = false
socket.close()
}
private fun sha256(value: String) =
MessageDigest.getInstance("SHA-256").digest(value.encodeToByteArray()).joinToString("") {
"%02x".format(it)
}
private fun xorDistance(left: String, right: String) =
left.zip(right.padEnd(left.length, '0')).count { it.first != it.second }
private fun String.encodeBase64() =
java.util.Base64.getUrlEncoder().encodeToString(encodeToByteArray())
private fun String.decodeBase64() = java.util.Base64.getUrlDecoder().decode(this).decodeToString()
}

View file

@ -0,0 +1,25 @@
package rip.crit.commonip.decentraland.kademlia
import java.util.concurrent.ConcurrentHashMap
class NodeNamer {
private val votes = ConcurrentHashMap<String, ConcurrentHashMap<String, MutableSet<String>>>()
fun observe(nodeId: String, observerId: String, alias: String) {
votes
.getOrPut(nodeId) { ConcurrentHashMap() }
.getOrPut(alias) { ConcurrentHashMap.newKeySet() }
.add(observerId)
}
fun name(nodeId: String): String =
votes[nodeId]?.maxByOrNull { it.value.size }?.takeIf { it.value.size >= 2 }?.key
?: deterministic(nodeId)
private fun deterministic(id: String): String {
val adjective = arrayOf("amber", "cedar", "delta", "ember", "flint", "grove", "harbor", "iris")
val noun = arrayOf("atlas", "beacon", "cinder", "drift", "echo", "forge", "glade", "horizon")
val number = id.take(8).toLong(16)
return "${adjective[(number % adjective.size).toInt()]}-${noun[((number / adjective.size) % noun.size).toInt()]}-${id.takeLast(4)}"
}
}

View file

@ -0,0 +1,58 @@
package rip.crit.commonip.decentraland.kademlia
import java.time.Instant
import java.util.concurrent.CopyOnWriteArrayList
enum class NodeState {
ONLINE,
DEGRADED,
OFFLINE,
}
data class NetworkNode(
val id: String,
val endpoint: String,
val state: NodeState,
val latencyMs: Long?,
val records: Int,
val lastSeen: Instant = Instant.now(),
)
data class NetworkActivity(
val at: Instant,
val nodeId: String,
val action: String,
val detail: String,
)
class NodeRegistry {
private val nodes = linkedMapOf<String, NetworkNode>()
private val activity = CopyOnWriteArrayList<NetworkActivity>()
@Synchronized
fun register(node: NetworkNode) {
nodes[node.id] = node
publish(node.id, "peer announced", "${node.endpoint} is ${node.state.name.lowercase()}")
}
@Synchronized
fun update(id: String, state: NodeState, latencyMs: Long? = null, records: Int? = null) {
val prior = nodes[id] ?: return
nodes[id] =
prior.copy(
state = state,
latencyMs = latencyMs ?: prior.latencyMs,
records = records ?: prior.records,
lastSeen = Instant.now(),
)
publish(id, "status changed", state.name.lowercase())
}
fun snapshot(): List<NetworkNode> = synchronized(this) { nodes.values.sortedBy { it.id } }
fun feed(limit: Int = 12): List<NetworkActivity> = activity.takeLast(limit).reversed()
fun publish(nodeId: String, action: String, detail: String) {
activity += NetworkActivity(Instant.now(), nodeId, action, detail)
}
}

View file

@ -0,0 +1,14 @@
package rip.crit.commonip.decentraland.kademlia
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
class NodeNamerTest {
@Test
fun `two peer votes override deterministic node name`() {
val namer = NodeNamer()
namer.observe("abcdef012345", "peer-a", "north")
namer.observe("abcdef012345", "peer-b", "north")
assertEquals("north", namer.name("abcdef012345"))
}
}

View file

@ -0,0 +1,3 @@
rootProject.name = "commonip-decentraland"
include("kademlia", "git", "chksum")

View file

@ -0,0 +1,33 @@
package rip.crit.commonip.decentraland
import java.nio.file.Path
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory
import org.apache.sshd.server.SshServer
import org.apache.sshd.server.auth.password.PasswordAuthenticator
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider
class SshDashboardService(
port: Int,
hostKey: Path,
username: String,
password: CharArray,
private val dashboard: () -> String,
) : AutoCloseable {
private val server =
SshServer.setUpDefaultServer().apply {
this.port = port
keyPairProvider = SimpleGeneratorHostKeyProvider(hostKey)
passwordAuthenticator = PasswordAuthenticator { user, candidate, _ ->
user == username && candidate.toCharArray().contentEquals(password)
}
fileSystemFactory = VirtualFileSystemFactory()
}
fun start() {
server.start()
}
fun render(): String = dashboard()
override fun close() = server.close()
}

View file

@ -0,0 +1,13 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
id("buildlogic.kotlin-library-conventions")
}
dependencies {
api(project(":packages:serialize"))
}

View file

@ -0,0 +1,235 @@
package rip.crit.commonip.ml
import kotlin.math.*
data class GeoLocation(
val latitude: Double?,
val longitude: Double?,
val confidence: Double = 1.0,
)
data class LocationEstimate(
val latitude: Double,
val longitude: Double,
val confidence: Double,
val sampleCount: Int,
val discardedCount: Int,
val spatialSpreadKm: Double,
)
data class Vector3D(val x: Double, val y: Double, val z: Double) {
fun normalize(): Vector3D {
val mag = sqrt(x * x + y * y + z * z)
return if (mag == 0.0) Vector3D(0.0, 0.0, 0.0) else Vector3D(x / mag, y / mag, z / mag)
}
fun toGeoLocation(confidence: Double = 1.0): GeoLocation {
val norm = this.normalize()
val lat = Math.toDegrees(asin(norm.z.coerceIn(-1.0, 1.0)))
val lon = Math.toDegrees(atan2(norm.y, norm.x))
return GeoLocation(lat, lon, confidence)
}
}
fun GeoLocation.toVector3D(): Vector3D {
val latRad = Math.toRadians(this.latitude!!)
val lonRad = Math.toRadians(this.longitude!!)
return Vector3D(
x = cos(latRad) * cos(lonRad),
y = cos(latRad) * sin(lonRad),
z = sin(latRad),
)
}
class LocationEstimator(
private val dbscanEpsKm: Double = 250.0,
private val dbscanMinPts: Int = 2,
private val medianIterations: Int = 20,
) {
fun estimate(observations: List<GeoLocation>): LocationEstimate? {
val validPoints = observations.filter { it.latitude != null && it.longitude != null }
if (validPoints.isEmpty()) return null
val clusteredPoints = dbscanCluster(validPoints, dbscanEpsKm, dbscanMinPts)
val selected = clusteredPoints.ifEmpty { validPoints }
val (estLat, estLon) = computeGeometricMedian(selected, medianIterations)
val distances = selected.map { distanceKm(estLat, estLon, it.latitude!!, it.longitude!!) }
val standardDistanceKm = sqrt(distances.sumOf { it * it } / selected.size)
val meanSampleConfidence = selected.map { max(it.confidence, 0.01) }.average()
val densityFactor = 1.0 / (1.0 + exp(-(selected.size - 3.0) / 2.0))
val spreadPenalty = 1.0 / (1.0 + (standardDistanceKm / 300.0))
val overallConfidence =
(meanSampleConfidence * densityFactor * spreadPenalty).coerceIn(0.0, 1.0)
return LocationEstimate(
latitude = estLat,
longitude = estLon,
confidence = overallConfidence,
sampleCount = selected.size,
discardedCount = validPoints.size - selected.size,
spatialSpreadKm = standardDistanceKm,
)
}
/** Finds the primary spatial cluster using DBSCAN logic. */
private fun dbscanCluster(
points: List<GeoLocation>,
epsKm: Double,
minPts: Int,
): List<GeoLocation> {
val visited = BooleanArray(points.size)
val clusters = mutableListOf<MutableList<GeoLocation>>()
for (i in points.indices) {
if (visited[i]) continue
visited[i] = true
val neighbors = getNeighbors(points, i, epsKm)
if (neighbors.size >= minPts) {
val currentCluster = mutableListOf(points[i])
val queue = neighbors.toMutableList()
var qIndex = 0
while (qIndex < queue.size) {
val pIndex = queue[qIndex++]
if (!visited[pIndex]) {
visited[pIndex] = true
val subNeighbors = getNeighbors(points, pIndex, epsKm)
if (subNeighbors.size >= minPts) {
queue.addAll(subNeighbors.filter { !queue.contains(it) })
}
}
if (!currentCluster.contains(points[pIndex])) {
currentCluster.add(points[pIndex])
}
}
clusters.add(currentCluster)
}
}
return clusters.maxByOrNull { cluster -> cluster.sumOf { max(it.confidence, 0.05) } }
?: emptyList()
}
private fun getNeighbors(points: List<GeoLocation>, centerIdx: Int, epsKm: Double): List<Int> {
val center = points[centerIdx]
val result = mutableListOf<Int>()
for (i in points.indices) {
if (
distanceKm(
center.latitude!!,
center.longitude!!,
points[i].latitude!!,
points[i].longitude!!,
) <= epsKm
) {
result.add(i)
}
}
return result
}
/** Weiszfeld's algorithm for finding the spatial geometric median (L1 median). */
private fun computeGeometricMedian(
points: List<GeoLocation>,
iterations: Int,
): Pair<Double, Double> {
if (points.size == 1) return Pair(points[0].latitude!!, points[0].longitude!!)
val weights = points.map { max(it.confidence, 0.05) }
val sumWeight = weights.sum()
var currentVector =
points
.zip(weights)
.map { (pt, w) ->
val vec = pt.toVector3D()
Vector3D(vec.x * w, vec.y * w, vec.z * w)
}
.reduce { acc, v ->
Vector3D(acc.x + v.x, acc.y + v.y, acc.z + v.z)
}
.let { Vector3D(it.x / sumWeight, it.y / sumWeight, it.z / sumWeight).normalize() }
var currentGeo = currentVector.toGeoLocation()
repeat(iterations) {
var sumX = 0.0
var sumY = 0.0
var sumZ = 0.0
var totalW = 0.0
for (i in points.indices) {
val ptVec = points[i].toVector3D()
val dist =
max(
distanceKm(
currentGeo.latitude!!,
currentGeo.longitude!!,
points[i].latitude!!,
points[i].longitude!!,
),
0.1,
)
val invWeight = weights[i] / dist
sumX += ptVec.x * invWeight
sumY += ptVec.y * invWeight
sumZ += ptVec.z * invWeight
totalW += invWeight
}
if (totalW > 0.0) {
currentVector = Vector3D(sumX / totalW, sumY / totalW, sumZ / totalW).normalize()
currentGeo = currentVector.toGeoLocation()
}
}
return Pair(currentGeo.latitude!!, currentGeo.longitude!!)
}
companion object {
fun distanceKm(aLat: Double, aLon: Double, bLat: Double, bLon: Double): Double {
val dLat = Math.toRadians(bLat - aLat)
val dLon = Math.toRadians(bLon - aLon)
val h =
sin(dLat / 2).pow(2) +
cos(Math.toRadians(aLat)) * cos(Math.toRadians(bLat)) * sin(dLon / 2).pow(2)
return 6371.0 * 2 * atan2(sqrt(h), sqrt(1 - h))
}
}
}
data class NodeTarget(
val nodeId: String,
val capacity: Int,
val latitude: Double?,
val longitude: Double?,
)
object LocationDispatcher {
fun select(
nodes: List<NodeTarget>,
estimate: LocationEstimate,
count: Int = 3,
): List<NodeTarget> =
nodes
.filter { it.capacity > 0 && it.latitude != null && it.longitude != null }
.sortedBy { node ->
val distance =
LocationEstimator.distanceKm(
estimate.latitude,
estimate.longitude,
node.latitude!!,
node.longitude!!,
)
distance / sqrt(node.capacity.toDouble())
}
.take(count)
}

View file

@ -0,0 +1,20 @@
package rip.crit.commonip.ml
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
class LocationEstimatorTest {
@Test
fun `estimator rejects distant outlier`() {
val result =
LocationEstimator()
.estimate(
listOf(
GeoLocation(40.0, -75.0, 0.9),
GeoLocation(40.1, -75.1, 0.9),
GeoLocation(-33.0, 151.0, 0.9),
)
)!!
assertEquals(1, result.discardedCount)
}
}

View file

@ -0,0 +1,13 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
id("buildlogic.kotlin-library-conventions")
}
dependencies {
api(project(":packages:serialize"))
}

View file

@ -0,0 +1,9 @@
package rip.crit.commonip.rawr.feeds
import rip.crit.commonip.serialize.GeoFeed
import rip.crit.commonip.serialize.GeoFeedCodec
import rip.crit.commonip.serialize.GeoFeedFormat
object RFC8805 {
fun parse(csv: String): GeoFeed = GeoFeedCodec.decode(csv, GeoFeedFormat.CSV)
}

View file

@ -0,0 +1,20 @@
package rip.crit.commonip.rawr.feeds
data class RFC9632(
val feedUrl: String,
val signatureUrl: String?,
val valid: Boolean,
val signer: String? = null,
) {
companion object {
fun fromDnsRecord(record: String): RFC9632? {
val values =
record
.split(';')
.map { it.trim() }
.associate { it.substringBefore('=').lowercase() to it.substringAfter('=', "") }
val url = values["geofeed"] ?: return null
return RFC9632(url, values["sig"], values["valid"] != "false", values["signer"])
}
}
}

View file

@ -0,0 +1,37 @@
package rip.crit.commonip.rawr.feeds
import rip.crit.commonip.rawr.types.Ingest
import rip.crit.commonip.serialize.GeoFeed
import rip.crit.commonip.serialize.GeoFeedCodec
import rip.crit.commonip.serialize.GeoFeedFormat
import rip.crit.commonip.serialize.GeoLocation
class RirFeed(private val name: String, private val loader: () -> String) : Ingest {
override fun ingest(): GeoFeed =
GeoFeedCodec.decode(loader(), GeoFeedFormat.CSV).let { feed ->
feed.copy(records = feed.records.map { it.copy(source = name) })
}
}
object DelegatedStats {
fun parse(payload: String, source: String): GeoFeed =
GeoFeed(
payload
.lineSequence()
.filter { it.isNotBlank() && !it.startsWith('#') && !it.startsWith("2|") }
.mapNotNull { line ->
val f = line.split('|')
if (f.size < 5 || f[2].lowercase() !in setOf("ipv4", "ipv6") || f[1] == "*") null
else {
val prefix =
if (f[2].lowercase() == "ipv4") "${f[3]}/${prefixFromCount(f[4])}"
else "${f[3]}/${f.getOrElse(4) { "48" }}"
GeoLocation(prefix, f[1].uppercase(), source = source, confidence = 0.25)
}
}
.toList()
)
private fun prefixFromCount(count: String): Int =
32 - Integer.numberOfTrailingZeros(count.toIntOrNull() ?: 1)
}

View file

@ -0,0 +1,8 @@
package rip.crit.commonip.rawr.providers
class AFRINIC(loader: (() -> String)? = null) :
RirProvider(
"AFRINIC",
"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
loader,
)

View file

@ -0,0 +1,8 @@
package rip.crit.commonip.rawr.providers
class APNIC(loader: (() -> String)? = null) :
RirProvider(
"APNIC",
"https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest",
loader,
)

View file

@ -0,0 +1,8 @@
package rip.crit.commonip.rawr.providers
class ARINIngest(loader: (() -> String)? = null) :
RirProvider(
"ARIN",
"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
loader,
)

View file

@ -0,0 +1,8 @@
package rip.crit.commonip.rawr.providers
class LACNIC(loader: (() -> String)? = null) :
RirProvider(
"LACNIC",
"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
loader,
)

View file

@ -0,0 +1,8 @@
package rip.crit.commonip.rawr.providers
class RIPENCC(loader: (() -> String)? = null) :
RirProvider(
"RIPE NCC",
"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
loader,
)

View file

@ -0,0 +1,49 @@
package rip.crit.commonip.rawr.providers
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import rip.crit.commonip.serialize.GeoFeed
import rip.crit.commonip.serialize.GeoFeedCodec
import rip.crit.commonip.serialize.GeoFeedFormat
enum class RirMetadataField(val keys: Set<String>) {
REMARKS(setOf("remarks", "remark")),
COMMENT(setOf("comment", "comments")),
}
class RirGeoFeedLoader(
private val source: String,
private val acceptedFields: Set<RirMetadataField>,
private val fetch: (String) -> String = RirGeoFeedLoader::httpGet,
) {
fun urls(registryObject: String): Set<String> =
registryObject
.lineSequence()
.map(String::trim)
.filter { ':' in it }
.filter { line ->
acceptedFields.any { field ->
line.substringBefore(':').trim().lowercase() in field.keys
}
}
.flatMap { urlPattern.findAll(it).map { match -> match.value } }
.toSet()
fun pull(registryObject: String): List<GeoFeed> =
urls(registryObject).map { url -> GeoFeedCodec.decode(fetch(url), GeoFeedFormat.CSV) }
private companion object {
val urlPattern = Regex("https?://[^\\s,;\"]+")
val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
fun httpGet(url: String): String =
client
.send(
HttpRequest.newBuilder(URI(url)).GET().build(),
HttpResponse.BodyHandlers.ofString(),
)
.body()
}
}

View file

@ -0,0 +1,31 @@
package rip.crit.commonip.rawr.providers
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import rip.crit.commonip.rawr.feeds.DelegatedStats
import rip.crit.commonip.rawr.types.Ingest
import rip.crit.commonip.serialize.GeoFeed
abstract class RirProvider(
private val name: String,
private val endpoint: String,
private val loader: (() -> String)? = null,
) : Ingest {
override fun ingest(): GeoFeed =
DelegatedStats.parse(
loader?.invoke()
?: client
.send(
HttpRequest.newBuilder(URI(endpoint)).GET().build(),
HttpResponse.BodyHandlers.ofString(),
)
.body(),
name,
)
private companion object {
val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
}
}

View file

@ -0,0 +1,36 @@
package rip.crit.commonip.rawr.types
import rip.crit.commonip.serialize.GeoFeed
import rip.crit.commonip.serialize.GeoLocation
interface Ingest {
fun ingest(): GeoFeed
fun lookupOne(address: String): GeoLocation? =
ingest()
.records
.filter { Cidr.contains(it.prefix, address) }
.maxByOrNull { Cidr.prefixLength(it.prefix) }
}
object Cidr {
fun prefixLength(cidr: String): Int = cidr.substringAfter('/').toIntOrNull() ?: -1
fun contains(cidr: String, address: String): Boolean {
val parts = cidr.split('/', limit = 2)
val bits = parts.getOrNull(1)?.toIntOrNull() ?: return false
if (bits !in 0..32) return false
val network = ipv4(parts[0]) ?: return false
val target = ipv4(address) ?: return false
val mask = if (bits == 0) 0 else (-1 shl (32 - bits))
return network and mask == target and mask
}
private fun ipv4(value: String): Int? {
val fields = value.split('.')
if (fields.size != 4) return null
return fields.fold(0) { result, field ->
field.toIntOrNull()?.takeIf { it in 0..255 }?.let { (result shl 8) or it } ?: return null
}
}
}

View file

@ -0,0 +1,13 @@
package rip.crit.commonip.rawr.providers
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
class RirProviderTest {
@Test
fun `arin parses delegated ipv4 fixture`() {
val feed = ARINIngest { "arin|US|ipv4|192.0.2.0|256|20260101|allocated" }.ingest()
assertEquals("192.0.2.0/24", feed.records.single().prefix)
assertEquals("US", feed.records.single().countryCode)
}
}

View file

@ -0,0 +1,9 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
id("buildlogic.kotlin-library-conventions")
}

View file

@ -0,0 +1,3 @@
package rip.crit.commonip.serialize
fun String.deserializeGeoFeed(format: GeoFeedFormat): GeoFeed = GeoFeedCodec.decode(this, format)

View file

@ -0,0 +1,34 @@
package rip.crit.commonip.serialize
import java.time.Instant
data class GeoLocation(
val prefix: String,
val countryCode: String,
val region: String? = null,
val city: String? = null,
val postalCode: String? = null,
val latitude: Double? = null,
val longitude: Double? = null,
val source: String = "unknown",
val observedAt: Instant = Instant.now(),
val confidence: Double = 0.5,
) {
init {
require('/' in prefix) { "prefix must be CIDR notation" }
require(countryCode.length == 2) { "countryCode must be ISO 3166-1 alpha-2" }
require(confidence in 0.0..1.0) { "confidence must be between 0 and 1" }
}
}
data class GeoFeed(
val records: List<GeoLocation>,
val generatedAt: Instant = Instant.now(),
val schemaVersion: String = "commonip/geofeed/v1",
)
enum class GeoFeedFormat {
CSV,
JSON,
XML,
}

View file

@ -0,0 +1,158 @@
package rip.crit.commonip.serialize
import java.time.Instant
object GeoFeedCodec {
fun encode(feed: GeoFeed, format: GeoFeedFormat): String =
when (format) {
GeoFeedFormat.CSV ->
feed.records.joinToString("\n") { record ->
listOf(
record.prefix,
record.countryCode,
record.region.orEmpty(),
record.city.orEmpty(),
record.postalCode.orEmpty(),
)
.joinToString(",") { csv(it) }
}
GeoFeedFormat.JSON ->
"{\"schema\":\"${feed.schemaVersion}\",\"generatedAt\":\"${feed.generatedAt}\",\"records\":[" +
feed.records.joinToString(",") { jsonRecord(it) } +
"]}"
GeoFeedFormat.XML ->
"<geofeed schema=\"${xml(feed.schemaVersion)}\" generatedAt=\"${feed.generatedAt}\">" +
feed.records.joinToString("") { record ->
"<location prefix=\"${xml(record.prefix)}\" country=\"${xml(record.countryCode)}\"${attr("region", record.region)}${attr("city", record.city)}${attr("postalCode", record.postalCode)}${attr("latitude", record.latitude)}${attr("longitude", record.longitude)} source=\"${xml(record.source)}\" observedAt=\"${record.observedAt}\" confidence=\"${record.confidence}\"/>"
} +
"</geofeed>"
}
fun decode(payload: String, format: GeoFeedFormat): GeoFeed =
when (format) {
GeoFeedFormat.CSV ->
GeoFeed(
payload
.lineSequence()
.filter { it.isNotBlank() && !it.trimStart().startsWith("#") }
.map { csvRecord(it) }
.toList()
)
GeoFeedFormat.JSON ->
GeoFeed(
jsonObjects(
payload.substringAfter("\"records\":[", "").substringBeforeLast("]", "")
)
.map { jsonToRecord(it) }
)
GeoFeedFormat.XML ->
GeoFeed(
Regex("<location\\s+([^>]+)/>")
.findAll(payload)
.map { xmlToRecord(it.groupValues[1]) }
.toList()
)
}
private fun csvRecord(line: String): GeoLocation {
val fields = splitCsv(line)
require(fields.size >= 2) { "RFC 8805 record needs a prefix and country" }
return GeoLocation(
fields[0],
fields[1].uppercase(),
fields.getOrNull(2).blankToNull(),
fields.getOrNull(3).blankToNull(),
fields.getOrNull(4).blankToNull(),
source = "rfc8805",
)
}
private fun jsonRecord(r: GeoLocation) =
"{" +
listOf(
"prefix" to r.prefix,
"countryCode" to r.countryCode,
"region" to r.region,
"city" to r.city,
"postalCode" to r.postalCode,
"latitude" to r.latitude,
"longitude" to r.longitude,
"source" to r.source,
"observedAt" to r.observedAt.toString(),
"confidence" to r.confidence,
)
.joinToString(",") { (k, v) ->
"\"$k\":" +
when (v) {
null -> "null"
is Number -> v.toString()
else -> "\"${json(v.toString())}\""
}
} +
"}"
private fun jsonToRecord(obj: String): GeoLocation {
fun value(name: String) =
Regex("\"$name\"\\s*:\\s*(null|\"((?:\\\\.|[^\"])*)\"|[-.0-9]+)").find(obj)?.let {
if (it.groupValues[1] == "null") null
else
it.groupValues[2]
.ifEmpty { it.groupValues[1] }
.replace("\\\"", "\"")
.replace("\\\\", "\\")
}
return GeoLocation(
value("prefix") ?: error("JSON record missing prefix"),
value("countryCode") ?: error("JSON record missing countryCode"),
value("region"),
value("city"),
value("postalCode"),
value("latitude")?.toDouble(),
value("longitude")?.toDouble(),
value("source") ?: "commonip",
value("observedAt")?.let(Instant::parse) ?: Instant.now(),
value("confidence")?.toDouble() ?: 0.5,
)
}
private fun xmlToRecord(attrs: String): GeoLocation {
fun value(name: String) =
Regex("$name=\"([^\"]*)\"").find(attrs)?.groupValues?.get(1)?.let {
it.replace("&quot;", "\"").replace("&amp;", "&")
}
return GeoLocation(
value("prefix") ?: error("XML record missing prefix"),
value("country") ?: error("XML record missing country"),
value("region"),
value("city"),
value("postalCode"),
value("latitude")?.toDouble(),
value("longitude")?.toDouble(),
value("source") ?: "commonip",
value("observedAt")?.let(Instant::parse) ?: Instant.now(),
value("confidence")?.toDouble() ?: 0.5,
)
}
private fun jsonObjects(raw: String): List<String> =
Regex("\\{(?:[^{}]|\\{[^{}]*})*}").findAll(raw).map { it.value }.toList()
private fun splitCsv(line: String): List<String> =
Regex("(?:^|,)(\"(?:[^\"]|\"\")*\"|[^,]*)")
.findAll(line)
.map { it.groupValues[1].removeSurrounding("\"").replace("\"\"", "\"") }
.toList()
private fun csv(value: String) =
if (value.any { it == ',' || it == '"' || it == '\n' }) "\"${value.replace("\"", "\"\"")}\""
else value
private fun json(value: String) = value.replace("\\", "\\\\").replace("\"", "\\\"")
private fun xml(value: String) = value.replace("&", "&amp;").replace("\"", "&quot;")
private fun attr(name: String, value: Any?) =
value?.let { " $name=\"${xml(it.toString())}\"" }.orEmpty()
private fun String?.blankToNull() = this?.takeIf { it.isNotBlank() }
}

View file

@ -0,0 +1,3 @@
package rip.crit.commonip.serialize
fun GeoFeed.serialize(format: GeoFeedFormat): String = GeoFeedCodec.encode(this, format)

View file

@ -0,0 +1,15 @@
package rip.crit.commonip.serialize
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
class GeoFeedCodecTest {
@Test
fun `RFC 8805 records round trip`() {
val feed = GeoFeed(listOf(GeoLocation("192.0.2.0/24", "US", "PA", "Philadelphia", "19106")))
val parsed =
GeoFeedCodec.decode(GeoFeedCodec.encode(feed, GeoFeedFormat.CSV), GeoFeedFormat.CSV)
assertEquals(feed.records.first().prefix, parsed.records.first().prefix)
assertEquals("Philadelphia", parsed.records.first().city)
}
}

View file

@ -0,0 +1,3 @@
rootProject.name = "commonip-packages"
include("decentraland", "serialize", "ml", "rawr")

View file

@ -0,0 +1,74 @@
import java.net.URI
import javax.xml.parsers.DocumentBuilderFactory
import org.w3c.dom.Element
import org.w3c.dom.Node
plugins {
id("buildlogic.kotlin-library-conventions")
}
dependencies {
api(project(":packages:serialize"))
implementation(project(":packages:rawr"))
implementation("commons-net:commons-net:3.11.1")
}
tasks.register("whois") {
group = "generation"
description = "Fetches latest WHOIS server mapping and generates WhoisServers.kt in source."
val outputDirectory = file("src/main/kotlin/rip/crit/commonip/proprietary")
outputs.file(outputDirectory.resolve("WhoisServers.kt"))
doLast {
val xmlUrl =
"https://raw.githubusercontent.com/whois-server-list/whois-server-list/refs/heads/master/whois-server-list.xml"
val inputStream = URI(xmlUrl).toURL().openStream()
val dbFactory = DocumentBuilderFactory.newInstance()
val dBuilder = dbFactory.newDocumentBuilder()
val xmlDoc = dBuilder.parse(inputStream)
xmlDoc.documentElement.normalize()
val serverMap = mutableMapOf<String, String>()
val domainNodes = xmlDoc.getElementsByTagName("domain")
for (i in 0 until domainNodes.length) {
val domainNode = domainNodes.item(i)
if (domainNode.nodeType == Node.ELEMENT_NODE) {
val domainElement = domainNode as Element
val domainName = domainElement.getAttribute("name")
val whoisServerNodes = domainElement.getElementsByTagName("whoisServer")
if (whoisServerNodes.length > 0) {
val whoisServerElement = whoisServerNodes.item(0) as Element
val host = whoisServerElement.getAttribute("host")
if (host.isNotBlank() && domainName.isNotBlank()) {
serverMap[domainName] = host
}
}
}
}
val mapEntries = StringBuilder()
serverMap.forEach { (tld, server) ->
mapEntries.append(" \"$tld\" to \"$server\",\n")
}
val outputFile = outputDirectory.resolve("WhoisServers.kt")
outputFile.parentFile.mkdirs()
outputFile.writeText(
"""
package rip.crit.commonip.proprietary
// Auto-generated file. Do not edit directly.
object WhoisServers {
val map: Map<String, String> = mapOf(
$mapEntries )
}
"""
.trimIndent()
)
println("Successfully generated ${outputFile.path} with ${serverMap.size} active TLD servers.")
}
}

View file

@ -0,0 +1,19 @@
package rip.crit.commonip.proprietary
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
fun interface HttpTransport {
fun get(url: String, headers: Map<String, String>): String
}
object DefaultHttpTransport : HttpTransport {
private val client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()
override fun get(url: String, headers: Map<String, String>): String {
val request = HttpRequest.newBuilder(URI(url)).apply { headers.forEach(::header) }.GET().build()
return client.send(request, HttpResponse.BodyHandlers.ofString()).body()
}
}

View file

@ -0,0 +1,18 @@
package rip.crit.commonip.proprietary
import rip.crit.commonip.rawr.types.Cidr
import rip.crit.commonip.serialize.GeoFeed
import rip.crit.commonip.serialize.GeoFeedCodec
import rip.crit.commonip.serialize.GeoFeedFormat
import rip.crit.commonip.serialize.GeoLocation
class OpenGeoFeed(private val transport: HttpTransport = DefaultHttpTransport) {
fun pull(url: String): GeoFeed =
GeoFeedCodec.decode(transport.get(url, emptyMap()), GeoFeedFormat.CSV)
fun lookup(url: String, address: String): GeoLocation? =
pull(url)
.records
.filter { Cidr.contains(it.prefix, address) }
.maxByOrNull { Cidr.prefixLength(it.prefix) }
}

View file

@ -0,0 +1,6 @@
package rip.crit.commonip.proprietary
class Rdap(private val transport: HttpTransport = DefaultHttpTransport) {
fun lookup(address: String): String =
transport.get("https://rdap.org/ip/$address", mapOf("Accept" to "application/rdap+json"))
}

View file

@ -0,0 +1,31 @@
package rip.crit.commonip.proprietary
import java.time.Duration
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Semaphore
class RequestGovernor(
private val minimumInterval: Duration = Duration.ofMillis(500),
parallelism: Int = 2,
) {
private val permits = Semaphore(parallelism)
private val lastRequest = ConcurrentHashMap<String, Long>()
fun <T> execute(provider: String, action: () -> T): T {
permits.acquire()
try {
synchronized(lastRequest) {
val wait =
minimumInterval.toMillis() - (System.currentTimeMillis() - (lastRequest[provider] ?: 0))
if (wait > 0) Thread.sleep(wait)
lastRequest[provider] = System.currentTimeMillis()
}
return action()
} finally {
permits.release()
}
}
fun <T> chunks(values: Collection<T>, size: Int = 100): Sequence<List<T>> =
values.asSequence().chunked(size)
}

View file

@ -0,0 +1,13 @@
package rip.crit.commonip.proprietary
import rip.crit.commonip.serialize.GeoFeed
import rip.crit.commonip.serialize.GeoFeedCodec
import rip.crit.commonip.serialize.GeoFeedFormat
class StarlinkGeoFeed(private val transport: HttpTransport = DefaultHttpTransport) {
companion object {
const val URL = "https://geoip.starlinkisp.net/feed.csv"
}
fun pull(): GeoFeed = GeoFeedCodec.decode(transport.get(URL, emptyMap()), GeoFeedFormat.CSV)
}

View file

@ -0,0 +1,17 @@
package rip.crit.commonip.proprietary
import org.apache.commons.net.whois.WhoisClient
class Whois(private val servers: Map<String, String> = WhoisServers.map) {
fun lookup(query: String): String {
val server =
servers[query.substringAfterLast('.', "").lowercase()] ?: servers.getValue("default")
val client = WhoisClient()
return try {
client.connect(server)
client.query("$query\r\n")
} finally {
if (client.isConnected) client.disconnect()
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,13 @@
package rip.crit.commonip.proprietary
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable
class LiveProviderTest {
@Test
@EnabledIfEnvironmentVariable(named = "COMMONIP_LIVE_TESTS", matches = "true")
fun `starlink live feed has records in isolated opt in test`() {
assertTrue(StarlinkGeoFeed().pull().records.isNotEmpty())
}
}

View file

@ -0,0 +1,19 @@
package rip.crit.commonip.proprietary
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
class ProviderTest {
@Test
fun `open geofeed uses injected fixture transport`() {
val feed =
OpenGeoFeed(HttpTransport { _, _ -> "203.0.113.0/24,US,CA,Los Angeles,90001" })
.pull("https://fixture.invalid/feed.csv")
assertEquals("Los Angeles", feed.records.single().city)
}
@Test
fun `starlink endpoint is opt in provider`() {
assertEquals("https://geoip.starlinkisp.net/feed.csv", StarlinkGeoFeed.URL)
}
}

12
scripts/commonip.service Normal file
View file

@ -0,0 +1,12 @@
[Unit]
Description=CommonIP node
After=network-online.target
[Service]
User=commonip
WorkingDirectory=/opt/commonip
ExecStart=/opt/commonip/commonip
Restart=on-failure
[Install]
WantedBy=multi-user.target

14
scripts/deploy-ssh.sh Executable file
View file

@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
deploy_host="${1:?host is required}"
deploy_user="${2:?user is required}"
deploy_path="${COMMONIP_REMOTE_PATH:-/opt/commonip}"
./gradlew :app:nativeImage
ssh "${deploy_user}@${deploy_host}" "sudo useradd --system --create-home --shell /usr/sbin/nologin commonip 2>/dev/null || true; sudo install -d -o commonip -g commonip ${deploy_path}"
scp app/build/native/commonip "${deploy_user}@${deploy_host}:/tmp/commonip"
ssh "${deploy_user}@${deploy_host}" "sudo install -o commonip -g commonip -m 0755 /tmp/commonip ${deploy_path}/commonip"
scp scripts/commonip.service "${deploy_user}@${deploy_host}:/tmp/commonip.service"
ssh "${deploy_user}@${deploy_host}" "sudo install -m 0644 /tmp/commonip.service /etc/systemd/system/commonip.service; sudo systemctl daemon-reload; sudo systemctl enable --now commonip"

13
settings.gradle.kts Normal file
View file

@ -0,0 +1,13 @@
pluginManagement {
includeBuild("build-logic")
}
plugins {
id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0"
}
rootProject.name = "commonip"
include("app", "proprietary", "packages")
include(":packages:serialize", ":packages:rawr", ":packages:ml", ":packages:decentraland")
include(":packages:decentraland:kademlia", ":packages:decentraland:git", ":packages:decentraland:chksum")