Implementation for #281

* The limit is enforced on 2 places
    1. with the `ConversationRepoListener`: required when a Conversation with more as the maximum allowed number of conversations is directly added to the Repository
    2. with a `$slice` command in the update of `appendMessage(..)` of the `ConversationRepository`
* The limit is configured by `smarti.storage.mongodb.maxConvMsg` the default is `5000`
    * configuration changes are applied only on the next update to an conversation. This is sufficient as this limit is intended to prevent Mongo Documents getting over the limit of 16MByte
* Added a UnitTest for the MongoRepository that validates this feature. For testing the limit is set to `50` messages

application/src/main/resources/application.properties

@@ -63,6 +63,9 @@ smarti.analysis.language=de
 #The number of messages analyzed for a conversation (-1 for all)
 smarti.analysis.conextSize=10
 
+##The maximum messages per conversation
+#see #281 - ensures that conversations to not exceed the max document size of Mongodb (16MByte)
+smarti.storage.mongodb.maxConvMsg=5000
 
 #enable/disable full rebuild of indexes on startup (default: true)
 #smarti.index.rebuildOnStartup=true

core/src/main/java/io/redlink/smarti/model/Conversation.java

@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
+
 import org.bson.types.ObjectId;
 import org.springframework.data.annotation.Id;
 import org.springframework.data.annotation.PersistenceConstructor;
@@ -70,7 +71,7 @@ public class Conversation {
     private User user = new User();
 
     @ApiModelProperty(required = true, value = "List of Messages")
-    private final List<Message> messages = new LinkedList<>();
+    private final List<Message> messages = new LinkedList<Message>();
 
 //    NOTE: removed with 0.7.0: Analysis is now stored in an own collection. Mainly because one
 //    conversation might have different analysis for clients with different configurations.
@@ -89,6 +90,7 @@ public class Conversation {
     @Indexed(sparse=false)
     private final Date deleted;
 
+
     public Conversation(){
         this(null, null, null);
     }

core/src/main/java/io/redlink/smarti/repositories/ConversationRepoListener.java

@@ -0,0 +1,46 @@
+package io.redlink.smarti.repositories;
+
+import java.util.Iterator;
+
+import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener;
+import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent;
+import org.springframework.stereotype.Component;
+
+import io.redlink.smarti.model.Conversation;
+import io.redlink.smarti.model.Message;
+
+/**
+ * Mongo application event listener for {@link Conversation}s that ensures that
+ * save and update operations to not store {@link Conversation}s with more as
+ * {@link ConversationRepository#DEFAULT_MAX_MESSAGES_PER_CONVERSATION} messages!
+ * 
+ * see <a href="https://github.com/redlink-gmbh/smarti/issues/281">#281</a> for details
+ * 
+ * @author Rupert Westenthaler
+ *
+ */
+@Component
+public class ConversationRepoListener extends AbstractMongoEventListener<Conversation> {
+
+
+    private final MongoConversationStorageConfig config;
+
+    public ConversationRepoListener(MongoConversationStorageConfig config){
+        this.config = config;
+    }
+
+    @Override
+    public void onBeforeConvert(BeforeConvertEvent<Conversation> event) {
+        Conversation conversation = event.getSource();
+        int numMsg = conversation.getMessages().size();
+        if(numMsg > config.getMaxConvMsg()){
+            int numRemove = numMsg - config.getMaxConvMsg();
+            //remove messages from the front (oldest) until the limit of messages is reached
+            Iterator<Message> it = conversation.getMessages().iterator();
+            for(int i = 0; i < numRemove && it.hasNext(); i++){
+                it.next();
+                it.remove();
+            }
+        }
+    }
+}

core/src/main/java/io/redlink/smarti/repositories/ConversationRepositoryImpl.java

@@ -32,6 +32,7 @@
 import org.bson.types.ObjectId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.domain.Sort.Direction;
 import org.springframework.data.mongodb.core.MongoTemplate;
@@ -56,15 +57,20 @@
  * Custom implementations of Conversation Repository
  *
  * @author Sergio Fernández
+ * @author Rupert Westenthaler
+ * @author Jakob Frank
  */
+@EnableConfigurationProperties(MongoConversationStorageConfig.class)
 public class ConversationRepositoryImpl implements ConversationRepositoryCustom {
 
     private final Logger log = LoggerFactory.getLogger(getClass());
-
+    private final MongoConversationStorageConfig config;
+
     private final MongoTemplate mongoTemplate;
 
-    public ConversationRepositoryImpl(MongoTemplate mongoTemplate) {
+    public ConversationRepositoryImpl(MongoTemplate mongoTemplate, MongoConversationStorageConfig config) {
         this.mongoTemplate = mongoTemplate;
+        this.config = config;
 
         /* see #findLegacyConversation */
         mongoTemplate.indexOps(Conversation.class)
@@ -106,8 +112,11 @@ public Conversation appendMessage(Conversation conversation, Message message) {
             query.addCriteria(Criteria.where("messages").size(conversation.getMessages().size()));
 
             update = new Update();
-            update.addToSet("messages", message)
-                    .currentDate("lastModified");
+            //NOTE: we need to enforce MAM MESSAGES PER CONVERSATION (#281)
+            update.push("messages")
+                .slice(config.getMaxConvMsg()*-1)
+                .each(message)
+                .currentDate("lastModified");
         } else { //conversation not present or already marked as deleted
             throw new NotFoundException(Conversation.class, conversation.getId());
         }

core/src/main/java/io/redlink/smarti/repositories/MongoConversationStorageConfig.java

@@ -0,0 +1,27 @@
+package io.redlink.smarti.repositories;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@ConfigurationProperties(prefix="smarti.storage.mongodb")
+public class MongoConversationStorageConfig {
+
+    /**
+     * MongoBB has a 16MByte document size limit. So we need to limit the number of messages
+     * stored with a conversation to ensure that we keep under that limit.
+     * 
+     * Limiting the number of messages to 5000 will work if we need less as 400chars to represent
+     * single messages.
+     */ //#2281
+    public static final int DEFAULT_MAX_MESSAGES_PER_CONVERSATION = 5000;
+
+    int maxConvMsg = DEFAULT_MAX_MESSAGES_PER_CONVERSATION;
+
+
+    public int getMaxConvMsg() {
+        return maxConvMsg;
+    }
+
+    public void setMaxConvMsg(int maxConvMsg) {
+        this.maxConvMsg = maxConvMsg;
+    }
+}

core/src/test/java/io/redlink/smarti/repositories/ConversationRepositoryTest.java

@@ -0,0 +1,97 @@
+package io.redlink.smarti.repositories;
+
+import java.util.Arrays;
+import java.util.Date;
+
+import org.bson.types.ObjectId;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
+import org.springframework.test.context.ContextConfiguration;
+
+import io.redlink.smarti.model.Conversation;
+import io.redlink.smarti.model.ConversationMeta.Status;
+import io.redlink.smarti.model.Message;
+import io.redlink.smarti.model.Message.Origin;
+import io.redlink.smarti.model.User;
+import io.redlink.smarti.test.SpringServiceTest;
+
+@ContextConfiguration(classes={ConversationRepoListener.class})
+@EnableMongoRepositories(basePackageClasses={ConversationRepository.class})
+@EnableAutoConfiguration
+public class ConversationRepositoryTest extends SpringServiceTest {
+
+
+    @Autowired
+    private ConversationRepository conversationRepo;
+
+
+    @Test
+    public void testMessageLimit() throws Exception {
+        ObjectId owner = new ObjectId();
+        Conversation conv = new Conversation();
+        conv.setOwner(owner);
+
+        conv.getContext().setDomain("test");
+        conv.getContext().setContextType("test");
+        conv.getContext().setEnvironment("test", "test");
+
+        User user0 = new User();
+        user0.setDisplayName("Test Dummy");
+        user0.setEmail("[email protected]");
+        user0.setHomeTown("Testhausen");
+        conv.setUser(user0);
+
+        User user1 = new User();
+        user1.setDisplayName("Maria Testament");
+        user1.setEmail("[email protected]");
+        user1.setHomeTown("Antesten");
+
+        conv.getMeta().setStatus(Status.New);
+        conv.getMeta().setProperty("test", Arrays.asList("test1","test2","test3"));
+
+        for(int i=0; i < 55 ; i++){
+            Message message = new Message("msg-"+i);
+            message.setOrigin(Origin.User);
+            message.setUser(i%2 == 0 ? user0 : user1);
+            message.setTime(new Date());
+            message.setContent("This is the " + (i + 1) + "message of this conversation");
+            conv.getMessages().add(message);
+        }
+
+        //now save the conversation in the Repository
+        Conversation created = conversationRepo.save(conv);
+        Assert.assertNotNull(created.getId());
+        //as the message limit is set to 50 we expect the first 5 Messages to be sliced
+        Assert.assertEquals(50, created.getMessages().size());
+        Assert.assertEquals("msg-5", conv.getMessages().get(0).getId());
+        Assert.assertEquals("msg-54", created.getMessages().get(created.getMessages().size() - 1).getId());
+
+        //now lets append a message
+        Message appended = new Message("appended-0");
+        appended.setOrigin(Origin.User);
+        appended.setUser(user1);
+        appended.setContent("This is the first appended Message");
+        Conversation updated = conversationRepo.appendMessage(created, appended);
+        Assert.assertEquals(50,  updated.getMessages().size());
+        Assert.assertEquals("msg-6", updated.getMessages().get(0).getId());
+        Assert.assertEquals("appended-0", updated.getMessages().get(updated.getMessages().size() - 1).getId());
+
+        //lets delete a message and append an other
+        Assert.assertTrue(conversationRepo.deleteMessage(updated.getId(), "msg-10"));
+        updated = conversationRepo.findOne(updated.getId());
+        Assert.assertEquals(49, updated.getMessages().size());
+        appended = new Message("appended-1");
+        appended.setOrigin(Origin.User);
+        appended.setUser(user0);
+        appended.setContent("This is the second appended Message");
+        updated = conversationRepo.appendMessage(updated, appended);
+        Assert.assertEquals(50,  updated.getMessages().size());
+        Assert.assertEquals("msg-6", updated.getMessages().get(0).getId());
+        Assert.assertEquals("appended-1", updated.getMessages().get(updated.getMessages().size() - 1).getId());
+
+    }
+
+}

core/src/test/resources/application-test.properties

@@ -19,3 +19,6 @@ spring.data.mongodb.database=smarti-test
 
 spring.jackson.serialization.INDENT_OUTPUT=true
 
+# For tests we limit the maximum number of messages per conversation to 50
+# so that we do not have to add 5000 messages to test this limit!
+smarti.storage.mongodb.maxConvMsg=50