diff --git a/smack-im/src/main/java/org/jivesoftware/smack/roster/Roster.java b/smack-im/src/main/java/org/jivesoftware/smack/roster/Roster.java
index 501d65023..94ab78210 100644
--- a/smack-im/src/main/java/org/jivesoftware/smack/roster/Roster.java
+++ b/smack-im/src/main/java/org/jivesoftware/smack/roster/Roster.java
@@ -382,12 +382,14 @@ public final class Roster extends Manager {
     private synchronized Map<Resourcepart, Presence> getOrCreatePresencesInternal(BareJid entity) {
         Map<Resourcepart, Presence> entityPresences = getPresencesInternal(entity);
         if (entityPresences == null) {
-            entityPresences = new ConcurrentHashMap<>();
             if (contains(entity)) {
+                entityPresences = new ConcurrentHashMap<>();
                 presenceMap.put(entity, entityPresences);
             }
             else {
-                nonRosterPresenceMap.put(entity, entityPresences);
+                LruCache<Resourcepart, Presence> nonRosterEntityPresences = new LruCache<>(32);
+                nonRosterPresenceMap.put(entity, nonRosterEntityPresences);
+                entityPresences = nonRosterEntityPresences;
             }
         }
         return entityPresences;