io.netty.handler.codec.http.multipart.FileUpload Java Examples

/**
 * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code parts}.
 * @param request the {@link HttpRequest} containing headers and other metadata about the request.
 * @param parts the {@link InMemoryFile}s that will form the parts of the request.
 * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code parts}.
 * @throws HttpPostRequestEncoder.ErrorDataEncoderException
 * @throws IOException
 */
private HttpPostRequestEncoder createEncoder(HttpRequest request, InMemoryFile[] parts)
    throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException {
  HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false);
  HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true);
  if (parts != null) {
    for (InMemoryFile part : parts) {
      FileUpload fileUpload =
          new MemoryFileUpload(part.name, part.name, "application/octet-stream", "", Charset.forName("UTF-8"),
              part.content.remaining());
      fileUpload.setContent(Unpooled.wrappedBuffer(part.content));
      encoder.addBodyHttpData(fileUpload);
    }
  }
  return encoder;
}
 

private void write(String placeId, Device d, FileUpload file) throws Exception {
   if (file.isCompleted()) {
      byte[] imageData = file.get();
      // TODO:  we should add check to make sure the jpg isn't corrupt
      if(imageData != null && imageData.length > 0) {
         UPLOAD_SIZE.update(imageData.length);
         try (Timer.Context context = UPLOAD_WRITE.time()) {
            storage.write(d.getId().toString(), imageData);
         }
      } else {
         UPLOAD_EMPTY.inc();
         logger.info("got an upload for camera {} at {} with a null or zero-length file", d.getId(), placeId);
      }
   } else {
      UPLOAD_INCOMPLETE.inc();
      logger.warn("got an incomplete upload for camera {}", d.getId());;
   }
}
 

protected DFHttpMultiData(HttpPostRequestDecoder reqDecoder) {
	this.reqDecoder = reqDecoder;
	List<InterfaceHttpData> lsReqData = reqDecoder.getBodyHttpDatas();
	if(lsReqData.isEmpty()){
		lsData = null;
		partNum = 0;
	}else{
		int tmpNum = 0;
		lsData = new ArrayList<>(lsReqData.size());
		for(InterfaceHttpData reqData : lsReqData){
			if(reqData.getHttpDataType() == HttpDataType.FileUpload){
				FileUpload fUp = (FileUpload) reqData;
				String tmpFile = fUp.getFilename();
				if(tmpFile == null || tmpFile.equals("")){
					continue;
				}
				DFHttpData data = new DFHttpData(fUp);
				lsData.add(data);
				++tmpNum;
			}
		}
		partNum = tmpNum;
	}
}
 

/**
 * Parse FileUpload to {@link FileItem}.
 *
 * @param fileUpload netty http file upload
 */
private void parseFileUpload(FileUpload fileUpload) throws IOException {
    if (!fileUpload.isCompleted()) {
        return;
    }
    FileItem fileItem = new FileItem();
    fileItem.setName(fileUpload.getName());
    fileItem.setFileName(fileUpload.getFilename());

    // Upload the file is moved to the specified temporary file,
    // because FileUpload will be release after completion of the analysis.
    // tmpFile will be deleted automatically if they are used.
    Path tmpFile = Files.createTempFile(
            Paths.get(fileUpload.getFile().getParent()), "blade_", "_upload");

    Path fileUploadPath = Paths.get(fileUpload.getFile().getPath());
    Files.move(fileUploadPath, tmpFile, StandardCopyOption.REPLACE_EXISTING);

    fileItem.setFile(tmpFile.toFile());
    fileItem.setPath(tmpFile.toFile().getPath());
    fileItem.setContentType(fileUpload.getContentType());
    fileItem.setLength(fileUpload.length());

    fileItems.put(fileItem.getName(), fileItem);
}
 

private void parseHttpPostRequest(FullHttpRequest request) {
	HttpPostRequestDecoder decoder = null;
	try {
		decoder = new HttpPostRequestDecoder(request);
		for (InterfaceHttpData httpData : decoder.getBodyHttpDatas()) {
			HttpDataType _type = httpData.getHttpDataType();
			if (_type == HttpDataType.Attribute) {
				Attribute attribute = (Attribute) httpData;
				parseAttribute(attribute);
			} else if (_type == HttpDataType.FileUpload) {
				FileUpload upload = (FileUpload) httpData;
				multipartFiles.add(MultipartFileFactory.create(upload));
			}
		}
	} catch (Exception ex) {
		LogUtils.warn(ex.getMessage());
	} finally {
		// 注意这个地方，一定要调用destroy方法，如果不调用会导致内存泄漏
		if (decoder != null)
			decoder.destroy();
	}
}
 

/**
 * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code blobContent}.
 * @param request the {@link HttpRequest} containing headers and other metadata about the request.
 * @param blobContent the {@link ByteBuffer} that represents the content of the blob.
 * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code blobContent}.
 * @throws HttpPostRequestEncoder.ErrorDataEncoderException
 * @throws IOException
 */
private HttpPostRequestEncoder createEncoder(HttpRequest request, ByteBuffer blobContent)
    throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException {
  HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false);
  HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true);
  FileUpload fileUpload = new MemoryFileUpload(RestUtils.MultipartPost.BLOB_PART, RestUtils.MultipartPost.BLOB_PART,
      "application/octet-stream", "", Charset.forName("UTF-8"), blobContent.remaining());
  fileUpload.setContent(Unpooled.wrappedBuffer(blobContent));
  encoder.addBodyHttpData(fileUpload);
  return encoder;
}
 

private HttpResponseStatus readFileUploadData() throws IOException {
    while (decoder.hasNext()) {
        InterfaceHttpData data = decoder.next();
        if (data != null) {
            try {
                logger.info("BODY FileUpload: " + data.getHttpDataType().name() + ": " + data);
                if (data.getHttpDataType() == HttpDataType.FileUpload) {
                    FileUpload fileUpload = (FileUpload) data;
                    if (fileUpload.isCompleted()) {
                        requestProcessed = true;
                        String format = ImageStoreUtil.checkTemplateFormat(fileUpload.getFile().getAbsolutePath(), fileUpload.getFilename());
                        if(StringUtils.isNotBlank(format)) {
                            String errorString = "File type mismatch between the sent file and the actual content. Received: " + format;
                            logger.error(errorString);
                            responseContent.append(errorString);
                            storageResource.updateStateMapWithError(uuid, errorString);
                            return HttpResponseStatus.BAD_REQUEST;
                        }
                        String status = storageResource.postUpload(uuid, fileUpload.getFile().getName(), processTimeout);
                        if (status != null) {
                            responseContent.append(status);
                            storageResource.updateStateMapWithError(uuid, status);
                            return HttpResponseStatus.INTERNAL_SERVER_ERROR;
                        } else {
                            responseContent.append("upload successful.");
                            return HttpResponseStatus.OK;
                        }
                    }
                }
            } finally {
                data.release();
            }
        }
    }
    responseContent.append("received entity is not a file");
    return HttpResponseStatus.UNPROCESSABLE_ENTITY;
}
 

private void writeHttpData(InterfaceHttpData data) {
    try {
        InterfaceHttpData.HttpDataType dataType = data.getHttpDataType();
        if (dataType == InterfaceHttpData.HttpDataType.Attribute) {
            parseAttribute((Attribute) data);
        } else if (dataType == InterfaceHttpData.HttpDataType.FileUpload) {
            parseFileUpload((FileUpload) data);
        }
    } catch (IOException e) {
        log.error("Parse request parameter error", e);
    }
}
 

private void handleUploadFile(InterfaceHttpData data, Message uploadMessage) throws IOException{
	FileForm fileForm = (FileForm)uploadMessage.getBody();
       if(fileForm == null){
       	fileForm = new FileForm();
       	uploadMessage.setBody(fileForm);
       }
       
	if (data.getHttpDataType() == HttpDataType.Attribute) {
           Attribute attribute = (Attribute) data;
           fileForm.attributes.put(attribute.getName(), attribute.getValue());
           return;
	}
	
	if (data.getHttpDataType() == HttpDataType.FileUpload) {
           FileUpload fileUpload = (FileUpload) data;
           Http.FileUpload file = new Http.FileUpload();
           file.fileName = fileUpload.getFilename();
           file.contentType = fileUpload.getContentType();
           file.data = fileUpload.get(); 
           
           List<Http.FileUpload> uploads = fileForm.files.get(data.getName());
           if(uploads == null){
           	uploads = new ArrayList<Http.FileUpload>();
           	fileForm.files.put(data.getName(), uploads);
           }
           uploads.add(file);
	}
}
 

private HttpResponseStatus readFileUploadData() throws IOException {
    while (this.decoder.hasNext()) {
        final InterfaceHttpData data = this.decoder.next();
        if (data != null) {
            try {
                logger.info("BODY FileUpload: " + data.getHttpDataType().name() + ": " + data);
                if (data.getHttpDataType() == HttpDataType.FileUpload) {
                    final FileUpload fileUpload = (FileUpload) data;
                    if (fileUpload.isCompleted()) {
                        this.requestProcessed = true;
                        final String format = ImageStoreUtil.checkTemplateFormat(fileUpload.getFile().getAbsolutePath(), fileUpload.getFilename());
                        if (StringUtils.isNotBlank(format)) {
                            final String errorString = "File type mismatch between the sent file and the actual content. Received: " + format;
                            logger.error(errorString);
                            this.responseContent.append(errorString);
                            this.storageResource.updateStateMapWithError(this.uuid, errorString);
                            return HttpResponseStatus.BAD_REQUEST;
                        }
                        final String status = this.storageResource.postUpload(this.uuid, fileUpload.getFile().getName());
                        if (status != null) {
                            this.responseContent.append(status);
                            this.storageResource.updateStateMapWithError(this.uuid, status);
                            return HttpResponseStatus.INTERNAL_SERVER_ERROR;
                        } else {
                            this.responseContent.append("upload successful.");
                            return HttpResponseStatus.OK;
                        }
                    }
                }
            } finally {
                data.release();
            }
        }
    }
    this.responseContent.append("received entity is not a file");
    return HttpResponseStatus.UNPROCESSABLE_ENTITY;
}
 

@DataProvider(value = {
    "false  |   false",
    "false  |   true",
    "true   |   false",
    "true   |   true",
}, splitBy = "\\|")
@Test
public void getMultipartParts_works_as_expected_with_known_valid_data(
    boolean httpVersionIsNull, boolean httpMethodIsNull
) throws IOException {
    // given
    RequestInfoImpl<?> requestInfo = RequestInfoImpl.dummyInstanceForUnknownRequests();
    Whitebox.setInternalState(requestInfo, "isMultipart", true);
    Whitebox.setInternalState(requestInfo, "contentCharset", CharsetUtil.UTF_8);
    Whitebox.setInternalState(requestInfo, "protocolVersion", (httpVersionIsNull) ? null : HttpVersion.HTTP_1_1);
    Whitebox.setInternalState(requestInfo, "method", (httpMethodIsNull) ? null : HttpMethod.POST);
    requestInfo.isCompleteRequestWithAllChunks = true;
    requestInfo.rawContentBytes = KNOWN_MULTIPART_DATA_BODY.getBytes(CharsetUtil.UTF_8);
    requestInfo.getHeaders().set("Content-Type", KNOWN_MULTIPART_DATA_CONTENT_TYPE_HEADER);

    // when
    List<InterfaceHttpData> result = requestInfo.getMultipartParts();

    // then
    assertThat(result, notNullValue());
    assertThat(result.size(), is(1));
    InterfaceHttpData data = result.get(0);
    assertThat(data, instanceOf(FileUpload.class));
    FileUpload fileUploadData = (FileUpload)data;
    assertThat(fileUploadData.getName(), is(KNOWN_MULTIPART_DATA_NAME));
    assertThat(fileUploadData.getFilename(), is(KNOWN_MULTIPART_DATA_FILENAME));
    assertThat(fileUploadData.getString(CharsetUtil.UTF_8), is(KNOWN_MULTIPART_DATA_ATTR_UUID));
}
 

@Override
public @NotNull CompletableFuture<ResponseInfo<String>> execute(
    @NotNull RequestInfo<String> request,
    @NotNull Executor longRunningTaskExecutor,
    @NotNull ChannelHandlerContext ctx
) {
    List<String> hashesFound = new ArrayList<>();

    for (InterfaceHttpData multipartData : request.getMultipartParts()) {
        String name = multipartData.getName();
        byte[] payloadBytes;
        try {
            payloadBytes = ((HttpData)multipartData).get();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        String filename = null;
        switch (multipartData.getHttpDataType()) {
            case Attribute:
                // Do nothing - filename stays null
                break;
            case FileUpload:
                filename = ((FileUpload)multipartData).getFilename();
                break;
            default:
                throw new RuntimeException("Unsupported multipart type: " + multipartData.getHttpDataType().name());
        }

        hashesFound.add(getHashForMultipartPayload(name, filename, payloadBytes));
    }

    return CompletableFuture.completedFuture(ResponseInfo.newBuilder(StringUtils.join(hashesFound, ",")).build());
}
 

public static MultipartFile create(FileUpload fileUpload) {
	try {
		MultipartFile _file = new MultipartFile();
		_file.setContent(fileUpload.get());
		_file.setContentType(fileUpload.getContentType());
		_file.setSize(fileUpload.length());
		_file.setName(fileUpload.getFilename());
		return _file;
	} catch (Exception ex) {
		// ignore exception
	}
	return null;
}
 

private void doPost(ChannelHandlerContext ctx, SessionContext sctx) {
	WebRequest webRequest = getWebRequest();
	JSONObject data = webRequest.getData();
	try {
		while (postRequestDecoder.hasNext()) {
			InterfaceHttpData httpData = postRequestDecoder.next();
			try {
				if (httpData.getHttpDataType() == HttpDataType.Attribute || 
						httpData.getHttpDataType() == HttpDataType.InternalAttribute) {
					Attribute attribute = (Attribute) httpData;
					data.put(attribute.getName(), attribute.getValue());
				} else if (httpData.getHttpDataType() == HttpDataType.FileUpload) {
					FileUpload fileUpload = (FileUpload) httpData;
					if (fileUpload.isCompleted()) {
						webRequest.getFileUploadMap().put(fileUpload.getName(), new WebFileUpload(fileUpload));
					} else {
						log.error("fileUpload not complete name[{}]", fileUpload.getName());
					}
				}
			} catch (Exception e) {
				log.error(e.getMessage(), e);
			} finally {
				postRequestDecoder.removeHttpDataFromClean(httpData);
				httpData.release();
			}
		}
	} catch (EndOfDataDecoderException ignored) {
		
	}
	if (webRequest.isDynamic()) {
		doWork(ctx, sctx, webRequest);
	} else {
		doFile(ctx, sctx, webRequest);
	}
}
 

@Override
public void sendResponse(FullHttpRequest req, ChannelHandlerContext ctx) throws Exception {
   long startTime = System.nanoTime();

   HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(req);
   try {
      String place = null;
      int num = 0;
      while(decoder.hasNext()) {
         num++;

         InterfaceHttpData httpData = decoder.next();
         if(httpData.getHttpDataType() == HttpDataType.Attribute && httpData.getName().equalsIgnoreCase("place")) {
            place = ((Attribute) httpData).getValue();
         } else if(httpData.getHttpDataType() == HttpDataType.FileUpload) {
            String camProtAddr = URLDecoder.decode(httpData.getName(), "UTF-8");
            Device d = findCamera(camProtAddr);
            if(d == null) {
               UPLOAD_UNKNOWN.inc();
               logger.warn("ignoring preview upload for non-existent camera {}", camProtAddr);
               continue;
            }
            write(place, d, (FileUpload) httpData);
         }
      }
      HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
      ChannelFuture future = ctx.writeAndFlush(response);
      if(!HttpHeaders.isKeepAlive(req)) {
         future.addListener(ChannelFutureListener.CLOSE);
      }

      UPLOAD_NUM.update(num);
      UPLOAD_SUCCESS.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
   } catch (Exception ex) {
      UPLOAD_FAIL.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
   } finally {
      decoder.cleanFiles();
   }
}
 

public void init(Channel channel, FullHttpRequest request, RouteResult<RouteAction> routeResult, boolean enableCookies) {
    this.channel = channel;
    this.request = request;
    this.routeResult = routeResult;
    this.enableCookies = enableCookies;

    if (this.method() == HttpMethod.POST) {
        try { 
            HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(request);
            decoder.offer(request);
            List<InterfaceHttpData> paramsList = decoder.getBodyHttpDatas();
            for (InterfaceHttpData httpData : paramsList) {
            	if (httpData.getHttpDataType() == HttpDataType.Attribute) {
            		Attribute data = (Attribute) httpData;
            		postMaps.put(data.getName(), data.getValue());
            	} else if (httpData.getHttpDataType() == HttpDataType.FileUpload) {
            		MixedFileUpload fileUpload = (MixedFileUpload) httpData;
            		this.fileUpload = fileUpload;
            	} else {
            		LOGGER.error("not support http data type. type={}", httpData.getHttpDataType());
            	}
            }
        } catch (Exception ex) {
            LOGGER.error("{}", ex);
        }
    }

    if (enableCookies) {
        List<String> cookiesList = request.headers().getAll(HttpHeaderNames.COOKIE);
        cookiesList.forEach(h -> ServerCookieDecoder.STRICT.decode(h).forEach(c -> cookieMaps.put(c.name(), c)));
    }
}
 

WebFileUpload(FileUpload fileUpload) {
	fileName = fileUpload.getFilename();
	byteBuf = fileUpload.content();
}
 

public ServletFilePart(FileUpload fileUpload,ResourceManager resourceManager) {
    this.fileUpload = fileUpload;
    this.resourceManager = resourceManager;
}
 

protected DFHttpData(FileUpload fileUp) {
	this.fileUp = fileUp;
}
 

/**
 * Processes a single decoded part in a multipart request. Exposes the data in the part either through the channel
 * itself (if it is the blob part) or via {@link #getArgs()}.
 * @param part the {@link InterfaceHttpData} that needs to be processed.
 * @throws RestServiceException if the request channel is closed, if there is more than one part of the same name, if
 *                              the size obtained from the headers does not match the actual size of the blob part or
 *                              if {@code part} is not of the expected type ({@link FileUpload}).
 */
private void processPart(InterfaceHttpData part) throws RestServiceException {
  if (part.getHttpDataType() == InterfaceHttpData.HttpDataType.FileUpload) {
    FileUpload fileUpload = (FileUpload) part;
    if (fileUpload.getName().equals(RestUtils.MultipartPost.BLOB_PART)) {
      // this is actual data.
      if (hasBlob) {
        nettyMetrics.repeatedPartsError.inc();
        throw new RestServiceException("Request has more than one " + RestUtils.MultipartPost.BLOB_PART,
            RestServiceErrorCode.BadRequest);
      } else {
        hasBlob = true;
        if (getSize() != -1 && fileUpload.length() != getSize()) {
          nettyMetrics.multipartRequestSizeMismatchError.inc();
          throw new RestServiceException(
              "Request size [" + fileUpload.length() + "] does not match Content-Length [" + getSize() + "]",
              RestServiceErrorCode.BadRequest);
        } else {
          contentLock.lock();
          try {
            if (isOpen()) {
              requestContents.add(new DefaultHttpContent(ReferenceCountUtil.retain(fileUpload.content())));
              blobBytesReceived.set(fileUpload.content().capacity());
            } else {
              nettyMetrics.multipartRequestAlreadyClosedError.inc();
              throw new RestServiceException("Request is closed", RestServiceErrorCode.RequestChannelClosed);
            }
          } finally {
            contentLock.unlock();
          }
        }
      }
    } else {
      // this is any kind of data. (For ambry, this will be user metadata).
      // TODO: find a configurable way of rejecting unexpected file parts.
      String name = fileUpload.getName();
      if (allArgs.containsKey(name)) {
        nettyMetrics.repeatedPartsError.inc();
        throw new RestServiceException("Request already has a component named " + name,
            RestServiceErrorCode.BadRequest);
      } else {
        ByteBuffer buffer = ByteBuffer.allocate(fileUpload.content().readableBytes());
        // TODO: Possible optimization - Upgrade ByteBufferReadableStreamChannel to take a list of ByteBuffer. This
        // TODO: will avoid the copy.
        fileUpload.content().readBytes(buffer);
        buffer.flip();
        allArgs.put(name, buffer);
      }
    }
  } else {
    nettyMetrics.unsupportedPartError.inc();
    throw new RestServiceException("Unexpected HTTP data", RestServiceErrorCode.BadRequest);
  }
}
 

public FileUpload fileUpload() {
	return fileUpload;
}