diff --git a/configure.ac b/configure.ac
index caf76e7a..06b5d1de 100644
--- a/configure.ac
+++ b/configure.ac
@@ -165,12 +165,6 @@ case "$host" in
     # gcc: now: linker input file unused because linking not done
     use_ro_relocations=no
     use_immediate_binding=no
-    if [ test "x$CPPFLAGS" = "x" ] ; then
-        CPPFLAGS="-I/usr/local/include -I/usr/local/include/gpgme"
-    fi
-    if [ test "x$LDFLAGS" = "x" ] ; then
-        LDFLAGS="-L/usr/local/lib"
-    fi
     ;;
 esac
 
@@ -480,6 +474,21 @@ AS_IF([test "x$GPG_EXE" != x],
   ], [ gpg_exe="(not found)"]
 )
 
+if [test "$have_gpgme" = "yes" ]; then
+case "$host" in
+*-*-linux*)
+    ;;
+*-*-freebsd*)
+    if [ test "x$CPPFLAGS" = "x" ] ; then
+        CPPFLAGS="-I/usr/local/include -I/usr/local/include/gpgme"
+    fi
+    if [ test "x$LDFLAGS" = "x" ] ; then
+        LDFLAGS="-L/usr/local/lib"
+    fi
+    ;;
+esac
+fi
+
 dnl Check for libpcap, gdbm (or ndbm) if we are building the server component
 dnl
 AS_IF([test "$want_server" = yes], [