diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 9ade60b972..06f21485c5 100644
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -212,21 +212,12 @@ int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
     /* TMATCH list can be ignored, it contains TAGs and
      * tags are compatible to IP-only. */
 
-    IPOnlyCIDRItem *cidr_item;
-    cidr_item = s->CidrSrc;
-    while (cidr_item != NULL) {
-        if (cidr_item->negated)
-            return 0;
-
-        cidr_item = cidr_item->next;
-    }
-    cidr_item = s->CidrDst;
-    while (cidr_item != NULL) {
-        if (cidr_item->negated)
-            return 0;
-
-        cidr_item = cidr_item->next;
-    }
+    /* if any of the addresses uses negation, we don't support
+     * it in ip-only */
+    if (s->init_data->src_contains_negation)
+        return 0;
+    if (s->init_data->dst_contains_negation)
+        return 0;
 
     SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH];
     if (sm == NULL)
diff --git a/src/detect-parse.c b/src/detect-parse.c
index 754f0aade7..44b7ee66bf 100644
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -765,6 +765,9 @@ static int SigParseAddress(DetectEngineCtx *de_ctx,
         if (strcasecmp(addrstr, "any") == 0)
             s->flags |= SIG_FLAG_SRC_ANY;
 
+        s->init_data->src_contains_negation =
+            (strchr(addrstr, '!') != NULL);
+
         s->init_data->src = DetectParseAddress(de_ctx, addrstr);
         if (s->init_data->src == NULL)
             goto error;
@@ -772,6 +775,9 @@ static int SigParseAddress(DetectEngineCtx *de_ctx,
         if (strcasecmp(addrstr, "any") == 0)
             s->flags |= SIG_FLAG_DST_ANY;
 
+        s->init_data->dst_contains_negation =
+            (strchr(addrstr, '!') != NULL);
+
         s->init_data->dst = DetectParseAddress(de_ctx, addrstr);
         if (s->init_data->dst == NULL)
             goto error;
diff --git a/src/detect.h b/src/detect.h
index baba84b028..3d41697fb5 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -450,6 +450,11 @@ typedef struct SignatureInitData_ {
      *  have the SIGMATCH_HANDLE_NEGATION flag set. */
     bool negated;
 
+    /* track if we saw any negation in the addresses. If so, we
+     * skip it for ip-only */
+    bool src_contains_negation;
+    bool dst_contains_negation;
+
     /* used to hold flags that are used during init */
     uint32_t init_flags;
     /* coccinelle: SignatureInitData:init_flags:SIG_FLAG_INIT_ */