Skip to content

Commit

Permalink
Fix UCME (#118)
Browse files Browse the repository at this point in the history
  • Loading branch information
@devmotion
devmotion committed Jan 21, 2022
1 parent 8ba7021 commit 967c58e
Show file tree
Hide file tree
Showing 7 changed files with 50 additions and 48 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "CalibrationErrors"
uuid = "33913031-fe46-5864-950f-100836f47845"
authors = ["David Widmann <[email protected]>"]
version = "0.5.21"
version = "0.5.22"

[deps]
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
Expand Down
17 changes: 6 additions & 11 deletions src/ucme.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,8 @@ function unsafe_ucme_eval(
return res
end
function unsafe_ucme_eval(kernel::Kernel, p::Real, y::Bool, testp::Real, testy::Bool)
noty = !y
return (y - p) * kernel((p, y), (testp, testy)) +
(noty - p) * kernel((p, noty), (testp, testy))
return (kernel((p, true), (testp, testy)) - kernel((p, false), (testp, testy))) *
(y - p)
end

function unsafe_ucme_eval(kernel::KernelTensorProduct, p, y, testp, testy)
Expand Down Expand Up @@ -126,15 +125,12 @@ function unsafe_ucme_eval_targets(
testp::AbstractVector{<:Real},
testy::Integer,
)
res = sum(((z == y) - pz) * kernel(z, testy) for (z, pz) in enumerate(p))
return res
return sum(((z == y) - pz) * kernel(z, testy) for (z, pz) in enumerate(p))
end
function unsafe_ucme_eval_targets(
kernel::Kernel, p::Real, y::Bool, testp::Real, testy::Bool
)
noty = !y
res = (y - p) * kernel(y, testy) + (noty - p) * kernel(noty, testy)
return res
return (kernel(true, testy) - kernel(false, testy)) * (y - p)
end

function unsafe_ucme_eval_targets(
Expand All @@ -144,11 +140,10 @@ function unsafe_ucme_eval_targets(
testp::AbstractVector{<:Real},
testy::Integer,
)
@inbounds res = (y == testy) - p[testy]
return res
return @inbounds (y == testy) - p[testy]
end
function unsafe_ucme_eval_targets(
kernel::WhiteKernel, p::Real, y::Bool, testp::Real, testy::Bool
)
return (2 * testy - 1) * (y - p)
return (testy - !testy) * (y - p)
end
6 changes: 3 additions & 3 deletions test/binning/medianvariance.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
@test sum(bin -> bin.nsamples, bins) == nsamples
@test sum(bin -> bin.nsamples .* bin.mean_predictions, bins) ≈ sum(predictions)
@test sum(bin -> bin.nsamples .* bin.proportions_targets, bins) ≈
counts(targets, nclasses)
counts(targets, nclasses)
end

# set maximum number of bins
Expand All @@ -47,7 +47,7 @@
@test sum(bin -> bin.nsamples, bins) == nsamples
@test sum(bin -> bin.nsamples .* bin.mean_predictions, bins) ≈ sum(predictions)
@test sum(bin -> bin.nsamples .* bin.proportions_targets, bins) ≈
counts(targets, nclasses)
counts(targets, nclasses)
end
end

Expand Down Expand Up @@ -102,7 +102,7 @@
@test bins[i].nsamples == length(idxs)
@test bins[i].mean_predictions ≈ mean(predictions[idxs])
@test bins[i].proportions_targets ==
vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
end

bins = CalibrationErrors.perform(MedianVarianceBinning(3), predictions, targets)
Expand Down
4 changes: 2 additions & 2 deletions test/binning/uniform.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@
@test bins[i].nsamples == length(idxs)
@test bins[i].mean_predictions == mean(predictions[idxs])
@test bins[i].proportions_targets ==
vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
end

bins = CalibrationErrors.perform(UniformBinning(1), predictions, targets)
Expand Down Expand Up @@ -134,7 +134,7 @@
@test bins[i].nsamples == length(idxs)
@test bins[i].mean_predictions == mean(predictions[idxs])
@test bins[i].proportions_targets ==
vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
end

bins = CalibrationErrors.perform(UniformBinning(1), predictions, targets)
Expand Down
4 changes: 2 additions & 2 deletions test/deprecated.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@
for estimator in (ece, skce1, skce2, skce3, ucme)
estimate = estimator(predictions, targets)
@test @test_deprecated(calibrationerror(estimator, predictions, targets)) ==
estimate
estimate
@test @test_deprecated(calibrationerror(estimator, (predictions, targets))) ==
estimate
estimate
@test @test_deprecated(
calibrationerror(estimator, reduce(hcat, predictions), targets)
) == estimate
Expand Down
2 changes: 1 addition & 1 deletion test/skce/generic.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

@test unsafe_skce_eval(kernelfull ⊗ WhiteKernel(), pfull, yint, p̃full, ỹint) ≈ val
@test unsafe_skce_eval(kernelfull ⊗ WhiteKernel2(), pfull, yint, p̃full, ỹint) ≈
val
val
@test unsafe_skce_eval(
TensorProduct2(kernelfull, WhiteKernel()), pfull, yint, p̃full, ỹint
) ≈ val
Expand Down
63 changes: 35 additions & 28 deletions test/ucme.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,37 +93,44 @@
end

@testset "binary classification" begin
# probabilities and boolean targets
# probabilities and corresponding full categorical distribution
p, testp = rand(2)
y, testy = rand(Bool, 2)
scale = rand()
kernel = SqExponentialKernel() ∘ ScaleTransform(scale)
val = unsafe_ucme_eval(kernel ⊗ WhiteKernel(), p, y, testp, testy)
@test unsafe_ucme_eval(kernel ⊗ WhiteKernel2(), p, y, testp, testy) ≈ val
@test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel()), p, y, testp, testy) ≈
val
@test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel2()), p, y, testp, testy) ≈
val

# corresponding values and kernel for full categorical distribution
pfull = [p, 1 - p]
yint = 2 - y
testpfull = [testp, 1 - testp]
testyint = 2 - testy

# kernel for probabilities and corresponding one for full categorical distributions
scale = rand()
kernel = SqExponentialKernel() ∘ ScaleTransform(scale)
kernelfull = SqExponentialKernel() ∘ ScaleTransform(scale / sqrt(2))

@test unsafe_ucme_eval(
kernelfull ⊗ WhiteKernel(), pfull, yint, testpfull, testyint
) ≈ val
@test unsafe_ucme_eval(
kernelfull ⊗ WhiteKernel2(), pfull, yint, testpfull, testyint
) ≈ val
@test unsafe_ucme_eval(
TensorProduct2(kernelfull, WhiteKernel()), pfull, yint, testpfull, testyint
) ≈ val
@test unsafe_ucme_eval(
TensorProduct2(kernelfull, WhiteKernel2()), pfull, yint, testpfull, testyint
) ≈ val
# for different targets
for y in (true, false), testy in (true, false)
# check values for probabilities
val = unsafe_ucme_eval(kernel ⊗ WhiteKernel(), p, y, testp, testy)
@test unsafe_ucme_eval(kernel ⊗ WhiteKernel2(), p, y, testp, testy) ≈ val
@test unsafe_ucme_eval(
TensorProduct2(kernel, WhiteKernel()), p, y, testp, testy
) ≈ val
@test unsafe_ucme_eval(
TensorProduct2(kernel, WhiteKernel2()), p, y, testp, testy
) ≈ val

# check values for categorical distributions
yint = 2 - y
testyint = 2 - testy
@test unsafe_ucme_eval(
kernelfull ⊗ WhiteKernel(), pfull, yint, testpfull, testyint
) ≈ val
@test unsafe_ucme_eval(
kernelfull ⊗ WhiteKernel2(), pfull, yint, testpfull, testyint
) ≈ val
@test unsafe_ucme_eval(
TensorProduct2(kernelfull, WhiteKernel()), pfull, yint, testpfull, testyint
) ≈ val
@test unsafe_ucme_eval(
TensorProduct2(kernelfull, WhiteKernel2()), pfull, yint, testpfull, testyint
) ≈ val
end
end

@testset "multi-class classification" begin
Expand All @@ -140,8 +147,8 @@

@test unsafe_ucme_eval(kernel ⊗ WhiteKernel2(), p, y, testp, testy) ≈ val
@test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel()), p, y, testp, testy) ≈
val
val
@test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel2()), p, y, testp, testy) ≈
val
val
end
end

2 comments on commit 967c58e

@devmotion
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/52884

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.5.22 -m "<description of version>" 967c58e102a31fca217464e9e750078cc03b60d1
git push origin v0.5.22

Please sign in to comment.