Store file size in DB (#704)

sismics · Sep 14, 2023 · ab7ff25 · ab7ff25
1 parent eedf19a
commit ab7ff25
Show file tree

Hide file tree

Showing 29 changed files with 484 additions and 163 deletions.
diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java b/docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java
@@ -160,6 +160,7 @@ public File update(File file) {
         fileDb.setMimeType(file.getMimeType());
         fileDb.setVersionId(file.getVersionId());
         fileDb.setLatestVersion(file.isLatestVersion());
+        fileDb.setSize(file.getSize());
 
         return file;
     }
@@ -224,4 +225,12 @@ public List<File> getByVersionId(String versionId) {
         q.setParameter("versionId", versionId);
         return q.getResultList();
     }
+
+    public List<File> getFilesWithUnknownSize(int limit) {
+        EntityManager em = ThreadLocalContext.get().getEntityManager();
+        TypedQuery<File> q = em.createQuery("select f from File f where f.size = :size and f.deleteDate is null order by f.order asc", File.class);
+        q.setParameter("size", File.UNKNOWN_SIZE);
+        q.setMaxResults(limit);
+        return q.getResultList();
+    }
 }
diff --git a/docs-core/src/main/java/com/sismics/docs/core/event/FileDeletedAsyncEvent.java b/docs-core/src/main/java/com/sismics/docs/core/event/FileDeletedAsyncEvent.java
@@ -13,6 +13,8 @@ public class FileDeletedAsyncEvent extends UserEvent {
      */
     private String fileId;
 
+    private Long fileSize;
+
     public String getFileId() {
         return fileId;
     }
@@ -21,10 +23,19 @@ public void setFileId(String fileId) {
         this.fileId = fileId;
     }
 
+    public Long getFileSize() {
+        return fileSize;
+    }
+
+    public void setFileSize(Long fileSize) {
+        this.fileSize = fileSize;
+    }
+
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(this)
             .add("fileId", fileId)
+            .add("fileSize", fileSize)
             .toString();
     }
-}
+}
diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java
@@ -2,16 +2,19 @@
 
 import com.google.common.eventbus.AllowConcurrentEvents;
 import com.google.common.eventbus.Subscribe;
+import com.sismics.docs.core.dao.UserDao;
 import com.sismics.docs.core.event.FileDeletedAsyncEvent;
 import com.sismics.docs.core.model.context.AppContext;
+import com.sismics.docs.core.model.jpa.File;
+import com.sismics.docs.core.model.jpa.User;
 import com.sismics.docs.core.util.FileUtil;
 import com.sismics.docs.core.util.TransactionUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Listener on file deleted.
- * 
+ *
  * @author bgamard
  */
 public class FileDeletedAsyncListener {
@@ -22,7 +25,7 @@ public class FileDeletedAsyncListener {
 
     /**
      * File deleted.
-     * 
+     *
      * @param event File deleted event
      * @throws Exception e
      */
@@ -32,6 +35,24 @@ public void on(final FileDeletedAsyncEvent event) throws Exception {
         if (log.isInfoEnabled()) {
             log.info("File deleted event: " + event.toString());
         }
+        TransactionUtil.handle(() -> {
+            // Update the user quota
+            UserDao userDao = new UserDao();
+            User user = userDao.getById(event.getUserId());
+            if (user != null) {
+                Long fileSize = event.getFileSize();
+
+                if (fileSize.equals(File.UNKNOWN_SIZE)) {
+                    // The file size was not in the database, in this case we need to get from the unencrypted size.
+                    fileSize = FileUtil.getFileSize(event.getFileId(), user);
+                }
+
+                if (! fileSize.equals(File.UNKNOWN_SIZE)) {
+                    user.setStorageCurrent(user.getStorageCurrent() - fileSize);
+                    userDao.updateQuota(user);
+                }
+            }
+        });
 
         // Delete the file from storage
         FileUtil.delete(event.getFileId());

diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java
@@ -9,6 +9,7 @@
 import com.sismics.docs.core.listener.async.*;
 import com.sismics.docs.core.model.jpa.User;
 import com.sismics.docs.core.service.FileService;
+import com.sismics.docs.core.service.FileSizeService;
 import com.sismics.docs.core.service.InboxService;
 import com.sismics.docs.core.util.PdfUtil;
 import com.sismics.docs.core.util.indexing.IndexingHandler;
@@ -65,6 +66,11 @@ public class AppContext {
      */
     private FileService fileService;
 
+    /**
+     * File size service.
+     */
+    private FileSizeService fileSizeService;
+
     /**
      * Asynchronous executors.
      */
@@ -102,6 +108,11 @@ private void startUp() {
         inboxService.startAsync();
         inboxService.awaitRunning();
 
+        // Start file size service
+        fileSizeService = new FileSizeService();
+        fileSizeService.startAsync();
+        fileSizeService.awaitRunning();
+
         // Register fonts
         PdfUtil.registerFonts();
 
@@ -238,6 +249,10 @@ public void shutDown() {
             fileService.stopAsync();
         }
 
+        if (fileSizeService != null) {
+            fileSizeService.stopAsync();
+        }
+
         instance = null;
     }
 }
diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java b/docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java
@@ -88,6 +88,14 @@ public class File implements Loggable {
     @Column(name = "FIL_LATESTVERSION_B", nullable = false)
     private boolean latestVersion;
 
+    public static final Long UNKNOWN_SIZE = -1L;
+
+    /**
+     * Can be {@link File#UNKNOWN_SIZE} if the size has not been stored in the database when the file has been uploaded
+     */
+    @Column(name = "FIL_SIZE_N", nullable = false)
+    private Long size;
+
     /**
      * Private key to decrypt the file.
      * Not saved to database, of course.
@@ -204,6 +212,18 @@ public File setLatestVersion(boolean latestVersion) {
         return this;
     }
 
+    /**
+     * Can return {@link File#UNKNOWN_SIZE} if the file size is not stored in the database.
+     */
+    public Long getSize() {
+        return size;
+    }
+
+    public File setSize(Long size) {
+        this.size = size;
+        return this;
+    }
+
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(this)

diff --git a/docs-core/src/main/java/com/sismics/docs/core/service/FileSizeService.java b/docs-core/src/main/java/com/sismics/docs/core/service/FileSizeService.java
@@ -0,0 +1,78 @@
+package com.sismics.docs.core.service;
+
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.sismics.docs.core.dao.FileDao;
+import com.sismics.docs.core.dao.UserDao;
+import com.sismics.docs.core.model.jpa.File;
+import com.sismics.docs.core.model.jpa.User;
+import com.sismics.docs.core.util.FileUtil;
+import com.sismics.docs.core.util.TransactionUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Service that retrieve files sizes when they are not in the database.
+ */
+public class FileSizeService extends AbstractScheduledService {
+    /**
+     * Logger.
+     */
+    private static final Logger log = LoggerFactory.getLogger(FileSizeService.class);
+
+    public FileSizeService() {
+    }
+
+    @Override
+    protected void startUp() {
+        log.info("File size service starting up");
+    }
+
+    @Override
+    protected void shutDown() {
+        log.info("File size service shutting down");
+    }
+
+    private static final int BATCH_SIZE = 30;
+
+    @Override
+    protected void runOneIteration() {
+        try {
+            TransactionUtil.handle(() -> {
+                FileDao fileDao = new FileDao();
+                List<File> files = fileDao.getFilesWithUnknownSize(BATCH_SIZE);
+                for(File file : files) {
+                    processFile(file);
+                }
+                if(files.size() < BATCH_SIZE) {
+                    log.info("No more file to process, stopping the service");
+                    stopAsync();
+                }
+            });
+        } catch (Throwable e) {
+            log.error("Exception during file service iteration", e);
+        }
+    }
+
+    void processFile(File file) {
+        UserDao userDao = new UserDao();
+        User user = userDao.getById(file.getUserId());
+        if(user == null) {
+            return;
+        }
+
+        long fileSize = FileUtil.getFileSize(file.getId(), user);
+        if(fileSize != File.UNKNOWN_SIZE){
+            FileDao fileDao = new FileDao();
+            file.setSize(fileSize);
+            fileDao.update(file);
+        }
+    }
+
+    @Override
+    protected Scheduler scheduler() {
+        return Scheduler.newFixedDelaySchedule(0, 1, TimeUnit.MINUTES);
+    }
+}
diff --git a/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java b/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java
@@ -1,14 +1,11 @@
 package com.sismics.docs.core.service;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.AbstractScheduledService;
 import com.sismics.docs.core.constant.ConfigType;
 import com.sismics.docs.core.dao.TagDao;
 import com.sismics.docs.core.dao.criteria.TagCriteria;
 import com.sismics.docs.core.dao.dto.TagDto;
 import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
-import com.sismics.docs.core.model.jpa.Config;
 import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.Tag;
 import com.sismics.docs.core.util.ConfigUtil;

diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java
@@ -16,6 +16,9 @@
 import com.sismics.util.context.ThreadLocalContext;
 import com.sismics.util.io.InputStreamReaderThread;
 import com.sismics.util.mime.MimeTypeUtil;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.CountingInputStream;
+import org.apache.commons.io.output.NullOutputStream;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,7 +49,7 @@ public class FileUtil {
     /**
      * File ID of files currently being processed.
      */
-    private static Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());
+    private static final Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());
 
     /**
      * Optical character recognition on an image.
@@ -149,6 +152,7 @@ public static String createFile(String name, String previousFileId, Path unencry
         file.setName(StringUtils.abbreviate(name, 200));
         file.setMimeType(mimeType);
         file.setUserId(userId);
+        file.setSize(fileSize);
 
         // Get files of this document
         FileDao fileDao = new FileDao();
@@ -240,4 +244,31 @@ public static void endProcessingFile(String fileId) {
     public static boolean isProcessingFile(String fileId) {
         return processingFileSet.contains(fileId);
     }
+
+    /**
+     * Get the size of a file on disk.
+     *
+     * @param fileId the file id
+     * @param user   the file owner
+     * @return the size or -1 if something went wrong
+     */
+    public static long getFileSize(String fileId, User user) {
+        // To get the size we copy the decrypted content into a null output stream
+        // and count the copied byte size.
+        Path storedFile = DirectoryUtil.getStorageDirectory().resolve(fileId);
+        if (! Files.exists(storedFile)) {
+            log.debug("File does not exist " + fileId);
+            return File.UNKNOWN_SIZE;
+        }
+        try (InputStream fileInputStream = Files.newInputStream(storedFile);
+             InputStream inputStream = EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey());
+             CountingInputStream countingInputStream = new CountingInputStream(inputStream);
+        ) {
+            IOUtils.copy(countingInputStream, NullOutputStream.NULL_OUTPUT_STREAM);
+            return countingInputStream.getByteCount();
+        } catch (Exception e) {
+            log.debug("Can't find size of file " + fileId, e);
+            return File.UNKNOWN_SIZE;
+        }
+    }
 }
diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/format/FormatHandlerUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/format/FormatHandlerUtil.java
@@ -3,7 +3,6 @@
 import com.google.common.collect.Lists;
 import com.sismics.util.ClasspathScanner;
 
-import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
 /**

diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/format/TextPlainFormatHandler.java b/docs-core/src/main/java/com/sismics/docs/core/util/format/TextPlainFormatHandler.java
@@ -1,6 +1,5 @@
 package com.sismics.docs.core.util.format;
 
-import com.google.common.base.Charsets;
 import com.google.common.io.Closer;
 import com.lowagie.text.*;
 import com.lowagie.text.pdf.PdfWriter;

diff --git a/docs-core/src/main/resources/config.properties b/docs-core/src/main/resources/config.properties
@@ -1 +1 @@
-db.version=28
+db.version=29
diff --git a/docs-core/src/main/resources/db/update/dbupdate-029-0.sql b/docs-core/src/main/resources/db/update/dbupdate-029-0.sql
@@ -0,0 +1,2 @@
+alter table T_FILE add column FIL_SIZE_N bigint not null default -1;
+update T_CONFIG set CFG_VALUE_C = '29' where CFG_ID_C = 'DB_VERSION';