diff --git a/R/geom-.R b/R/geom-.R
index 6d4ed6fc55..d28ea80908 100644
--- a/R/geom-.R
+++ b/R/geom-.R
@@ -176,8 +176,7 @@ Geom <- ggproto("Geom",
     # Override mappings with params
     aes_params <- intersect(self$aesthetics(), names(params))
     check_aesthetics(params[aes_params], nrow(data))
-    data[aes_params] <- params[aes_params]
-    data
+    vec_cbind(data[setdiff(names(data), aes_params)], !!!params[aes_params])
   },
 
   # Most parameters for the geom are taken automatically from draw_panel() or
diff --git a/tests/testthat/test-geom-smooth.R b/tests/testthat/test-geom-smooth.R
index e71df88485..c8a7f3af32 100644
--- a/tests/testthat/test-geom-smooth.R
+++ b/tests/testthat/test-geom-smooth.R
@@ -4,7 +4,7 @@ test_that("data is ordered by x", {
   ps <- ggplot(df, aes(x, y))+
     geom_smooth(stat = "identity", se = FALSE)
 
-  expect_equal(layer_data(ps)[c("x", "y")], df[order(df$x), ])
+  expect_equal(layer_data(ps)[c("x", "y")], df[order(df$x), ], ignore_attr = TRUE)
 })
 
 test_that("geom_smooth works in both directions", {
diff --git a/tests/testthat/test-layer.R b/tests/testthat/test-layer.R
index b0507cf7ae..3a7de53d8c 100644
--- a/tests/testthat/test-layer.R
+++ b/tests/testthat/test-layer.R
@@ -140,6 +140,19 @@ test_that("layer warns for constant aesthetics", {
 
 # Data extraction ---------------------------------------------------------
 
+test_that("AsIs data passes unmodified", {
+  p <- ggplot() + geom_blank(aes(x = 1:2, y = 1:2))
+  ld <- layer_data(p + geom_point(aes(x = I(0.5), y = I(0.5))), 2)
+  expect_s3_class(ld$x, "AsIs")
+  expect_equal(ld$y, I(0.5))
+  ld <- layer_data(p + geom_point(x = I(0.5), y = I(0.5), data = mtcars), 2)
+  expect_s3_class(ld$x, "AsIs")
+  expect_equal(ld$y[1], I(0.5))
+  ld <- layer_data(p + annotate("point", x = I(0.5), y = I(0.5)), 2)
+  expect_s3_class(ld$x, "AsIs")
+  expect_equal(ld$y, I(0.5))
+})
+
 test_that("layer_data returns a data.frame", {
   l <- geom_point()
   expect_equal(l$layer_data(mtcars), unrowname(mtcars))