diff --git a/src/detect.c b/src/detect.c
index 32ba52dec7..8625780d4a 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -1298,6 +1298,14 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
         SCMutexLock(&p->flow->m);
         {
+            /* set the iponly stuff */
+            if (p->flow != NULL) {
+                if (p->flow->flags & FLOW_TOCLIENT_IPONLY_SET)
+                    p->flowflags |= FLOW_PKT_TOCLIENT_IPONLY_SET;
+                if (p->flow->flags & FLOW_TOSERVER_IPONLY_SET)
+                    p->flowflags |= FLOW_PKT_TOSERVER_IPONLY_SET;
+            }
+
             /* Get the stored sgh from the flow (if any). Make sure we're not using
              * the sgh for icmp error packets part of the same stream. */
             if (IP_GET_IPPROTO(p) == p->flow->proto) { /* filter out icmp */
diff --git a/src/flow.c b/src/flow.c
index b9216c8589..0409ba2db1 100644
--- a/src/flow.c
+++ b/src/flow.c
@@ -791,12 +791,6 @@ void FlowHandlePacket (ThreadVars *tv, Packet *p)
     /* update queue positions */
     FlowUpdateQueue(f);
 
-    /* set the iponly stuff */
-    if (f->flags & FLOW_TOCLIENT_IPONLY_SET)
-        p->flowflags |= FLOW_PKT_TOCLIENT_IPONLY_SET;
-    if (f->flags & FLOW_TOSERVER_IPONLY_SET)
-        p->flowflags |= FLOW_PKT_TOSERVER_IPONLY_SET;
-
     /*set the detection bypass flags*/
     if (f->flags & FLOW_NOPACKET_INSPECTION) {
         SCLogDebug("setting FLOW_NOPACKET_INSPECTION flag on flow %p", f);