com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement Java Examples

The following examples show how to use com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: InsertHandler.java    From dble with GNU General Public License v2.0 6 votes vote down vote up
@Override
public void handle(DumpFileContext context, SQLStatement sqlStatement) throws InterruptedException {
    MySqlInsertStatement insert = (MySqlInsertStatement) sqlStatement;
    SQLInsertStatement.ValuesClause valueClause;

    valuesHandler.preProcess(context);
    for (int i = 0; i < insert.getValuesList().size(); i++) {
        valueClause = insert.getValuesList().get(i);
        try {
            processIncrementColumn(context, valueClause.getValues());
            valuesHandler.process(context, valueClause.getValues(), i == 0);
        } catch (SQLNonTransientException e) {
            context.addError(e.getMessage());
        }
    }
    valuesHandler.postProcess(context);
}
 
Example #2
Source File: MySqlInsertParser.java    From baymax with Apache License 2.0 6 votes vote down vote up
/**
 * 注意:子查询,批量,分片
 * @param result
 */
@Override
public void parse(ParseResult result) {

    // 解析sql
    statement = parser.parseStatement();

    MySqlInsertStatement insert = (MySqlInsertStatement)statement;

    String tableName = StringUtil.removeBackquote(insert.getTableName().getSimpleName());

    result.addTable(tableName);

    // 不是分区表
    if (!BaymaxContext.isPartitionTable(tableName)){
        return;
    }

    if (isMultiInsert(insert)){
        // 批量插入
        throw new UnsupportedOperationException("分区表不支持insert into ...values (),()...或 insert into ...select.....形式的批量插入");
    }else {
        // 单条插入
        result.setCalculateUnits(parserSingleInsert(tableName, insert));
    }
}
 
Example #3
Source File: ExplainHandler.java    From dble with GNU General Public License v2.0 6 votes vote down vote up
private static boolean isInsertSeq(ServerConnection c, String stmt, SchemaConfig schema) throws SQLException {
    SQLStatementParser parser = new MySqlStatementParser(stmt);
    MySqlInsertStatement statement = (MySqlInsertStatement) parser.parseStatement();
    String schemaName = schema == null ? null : schema.getName();
    SQLExprTableSource tableSource = statement.getTableSource();
    SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.getSchemaInfo(c.getUser(), schemaName, tableSource);
    String tableName = schemaInfo.getTable();
    schema = schemaInfo.getSchemaConfig();
    TableConfig tableConfig = schema.getTables().get(tableName);
    if (tableConfig == null) {
        return false;
    } else if (tableConfig.isAutoIncrement()) {
        return true;
    }
    return false;
}
 
Example #4
Source File: DruidInsertParser.java    From dble with GNU General Public License v2.0 6 votes vote down vote up
private void parserChildTable(SchemaInfo schemaInfo, final RouteResultset rrs, MySqlInsertStatement insertStmt,
                              final ServerConnection sc, boolean isExplain) throws SQLNonTransientException {

    final SchemaConfig schema = schemaInfo.getSchemaConfig();
    String tableName = schemaInfo.getTable();
    final TableConfig tc = schema.getTables().get(tableName);
    if (isMultiInsert(insertStmt)) {
        String msg = "ChildTable multi insert not provided";
        LOGGER.info(msg);
        throw new SQLNonTransientException(msg);
    }
    String joinKey = tc.getJoinKey();
    int joinKeyIndex = getJoinKeyIndex(schemaInfo, insertStmt, joinKey);
    final String joinKeyVal = insertStmt.getValues().getValues().get(joinKeyIndex).toString();
    String realVal = StringUtil.removeApostrophe(joinKeyVal);
    final String sql = RouterUtil.removeSchema(statementToString(insertStmt), schemaInfo.getSchema());
    rrs.setStatement(sql);
    // try to route by ER parent partion key
    RouteResultset theRrs = routeByERParentKey(rrs, tc, realVal, schemaInfo);
    if (theRrs != null) {
        rrs.setFinishedRoute(true);
    } else {
        rrs.setFinishedExecute(true);
        fetchChildTableToRoute(tc, joinKeyVal, sc, schema, sql, rrs, isExplain);
    }
}
 
Example #5
Source File: DruidInsertParser.java    From dble with GNU General Public License v2.0 6 votes vote down vote up
private boolean parserNoSharding(ServerConnection sc, String contextSchema, SchemaInfo schemaInfo, RouteResultset rrs,
                                 MySqlInsertStatement insert) throws SQLException {
    String noShardingNode = RouterUtil.isNoSharding(schemaInfo.getSchemaConfig(), schemaInfo.getTable());
    if (noShardingNode != null) {
        // table with single datanode and has autoIncrement property
        TableConfig tbConfig = schemaInfo.getSchemaConfig().getTables().get(schemaInfo.getTable());
        if (tbConfig != null && tbConfig.isAutoIncrement()) {
            return false;
        }
        StringPtr noShardingNodePr = new StringPtr(noShardingNode);
        Set<String> schemas = new HashSet<>();
        if (insert.getQuery() != null) {
            SQLSelectStatement selectStmt = new SQLSelectStatement(insert.getQuery());
            if (!SchemaUtil.isNoSharding(sc, insert.getQuery().getQuery(), insert, selectStmt, contextSchema, schemas, noShardingNodePr)) {
                return false;
            }
        }
        routeToNoSharding(schemaInfo.getSchemaConfig(), rrs, schemas, noShardingNodePr);
        return true;
    }
    return false;
}
 
Example #6
Source File: DruidMysqlCreateTableTest.java    From Mycat2 with GNU General Public License v3.0 5 votes vote down vote up
private boolean isInsertHasSlot(String sql)
{
    MySqlStatementParser parser = new MySqlStatementParser(sql);
    MySqlInsertStatement insertStatement= (MySqlInsertStatement)parser.parseStatement();
 List<SQLExpr> cc= insertStatement.getColumns();
    for (SQLExpr sqlExpr : cc) {
        SQLIdentifierExpr c= (SQLIdentifierExpr) sqlExpr;
        if("_slot".equalsIgnoreCase(c.getName())   &&cc.size()==insertStatement.getValues().getValues().size())    return true;
    }
    return false;
}
 
Example #7
Source File: DruidParserFactory.java    From Mycat2 with GNU General Public License v3.0 5 votes vote down vote up
public static DruidParser create(SchemaConfig schema, SQLStatement statement, SchemaStatVisitor visitor)
{
    DruidParser parser = null;
    if (statement instanceof SQLSelectStatement)
    {
        if(schema.isNeedSupportMultiDBType())
        {
            parser = getDruidParserForMultiDB(schema, statement, visitor);

        }

        if (parser == null)
        {
            parser = new DruidSelectParser();
        }
    } else if (statement instanceof MySqlInsertStatement)
    {
        parser = new DruidInsertParser();
    } else if (statement instanceof MySqlDeleteStatement)
    {
        parser = new DruidDeleteParser();
    } else if (statement instanceof MySqlCreateTableStatement)
    {
        parser = new DruidCreateTableParser();
    } else if (statement instanceof MySqlUpdateStatement)
    {
        parser = new DruidUpdateParser();
    } else if (statement instanceof SQLAlterTableStatement)
    {
        parser = new DruidAlterTableParser();
    } else if (statement instanceof MySqlLockTableStatement) {
    	parser = new DruidLockTableParser();
    } else
    {
        parser = new DefaultDruidParser();
    }

    return parser;
}
 
Example #8
Source File: DruidInsertParser.java    From Mycat2 with GNU General Public License v3.0 5 votes vote down vote up
/**
 * 寻找拆分字段在 columnList中的索引
 * @param insertStmt
 * @param partitionColumn
 * @return
 */
private int getShardingColIndex(MySqlInsertStatement insertStmt,String partitionColumn) {
	int shardingColIndex = -1;
	for(int i = 0; i < insertStmt.getColumns().size(); i++) {
		if(partitionColumn.equalsIgnoreCase(StringUtil.removeBackquote(insertStmt.getColumns().get(i).toString()))) {//找到分片字段
			shardingColIndex = i;
			return shardingColIndex;
		}
	}
	return shardingColIndex;
}
 
Example #9
Source File: ServerSchemaStatVisitor.java    From dble with GNU General Public License v2.0 5 votes vote down vote up
@Override
public boolean visit(MySqlInsertStatement x) {
    SQLName sqlName = x.getTableName();
    if (sqlName != null) {
        currentTable = sqlName.toString();
    }
    return false;
}
 
Example #10
Source File: MycatSchemaStatVisitor.java    From Mycat2 with GNU General Public License v3.0 5 votes vote down vote up
public boolean visit(MySqlInsertStatement x) {
    SQLName sqlName=  x.getTableName();
    if(sqlName!=null)
    {
        String table = sqlName.toString();
        if(table.startsWith("`"))
        {
            table=table.substring(1,table.length()-1);
        }
        setCurrentTable(sqlName.toString());
    }
    return false;
}
 
Example #11
Source File: ParseUtil.java    From Mycat2 with GNU General Public License v3.0 5 votes vote down vote up
public static String changeInsertAddSlot(String sql,int slotValue)
{
    SQLStatementParser parser = new MycatStatementParser(sql);
    MySqlInsertStatement insert = (MySqlInsertStatement) parser.parseStatement();
    insert.getColumns().add(new SQLIdentifierExpr("_slot") );
    insert.getValues().getValues().add(new SQLIntegerExpr(slotValue))  ;
    return insert.toString();
}
 
Example #12
Source File: ExplainHandler.java    From Mycat2 with GNU General Public License v3.0 5 votes vote down vote up
private static boolean isMycatSeq(String stmt, SchemaConfig schema)
  {
      if(pattern.matcher(stmt).find()) {
	return true;
}
      SQLStatementParser parser =new MySqlStatementParser(stmt);
      MySqlInsertStatement statement = (MySqlInsertStatement) parser.parseStatement();
      String tableName=   statement.getTableName().getSimpleName();
      TableConfig tableConfig= schema.getTables().get(tableName.toUpperCase());
      if(tableConfig==null) {
	return false;
}
      if(tableConfig.isAutoIncrement())
      {
          boolean isHasIdInSql=false;
          String primaryKey = tableConfig.getPrimaryKey();
          List<SQLExpr> columns = statement.getColumns();
          for (SQLExpr column : columns)
          {
              String columnName = column.toString();
              if(primaryKey.equalsIgnoreCase(columnName))
              {
                  isHasIdInSql = true;
                  break;
              }
          }
          if(!isHasIdInSql) {
		return true;
	}
      }


      return false;
  }
 
Example #13
Source File: CompensateService.java    From txle with Apache License 2.0 5 votes vote down vote up
private String parseTableName(SQLStatement sqlStatement) {
    if (sqlStatement instanceof MySqlInsertStatement) {
        return ((MySqlInsertStatement) sqlStatement).getTableName().toString();
    } else if (sqlStatement instanceof MySqlDeleteStatement) {
        return ((MySqlDeleteStatement) sqlStatement).getTableName().toString();
    } else if (sqlStatement instanceof MySqlUpdateStatement) {
        return ((MySqlUpdateStatement) sqlStatement).getTableName().toString();
    }
    return "";
}
 
Example #14
Source File: AutoCompensateHandler.java    From txle with Apache License 2.0 5 votes vote down vote up
@Override
public void prepareCompensationAfterExecuting(PreparedStatement delegate, String executeSql, Map<String, Object> standbyParams) throws SQLException {
    String globalTxId = CurrentThreadOmegaContext.getGlobalTxIdFromCurThread();
    if (globalTxId == null || globalTxId.length() == 0) {
        return;
    }
    String localTxId = CurrentThreadOmegaContext.getLocalTxIdFromCurThread();
    if (localTxId == null || localTxId.length() == 0) {
        return;
    }

    // To parse SQL by SQLParser tools from Druid.
    MySqlStatementParser parser = new MySqlStatementParser(executeSql);
    SQLStatement sqlStatement = parser.parseStatement();
    if (sqlStatement instanceof MySqlSelectIntoStatement) {
        return;
    }

    if (standbyParams == null) {
        standbyParams = new HashMap<>(8);
    }

    String server = CurrentThreadOmegaContext.getServiceNameFromCurThread();

    // To set a relationship between localTxId and datSourceInfo, in order to determine to use the relative dataSource for localTxId when it need be compensated.
    DatabaseMetaData databaseMetaData = delegate.getConnection().getMetaData();
    String dburl = databaseMetaData.getURL(), dbusername = databaseMetaData.getUserName(), dbdrivername = databaseMetaData.getDriverName();
    DataSourceMappingCache.putLocalTxIdAndDataSourceInfo(localTxId, dburl, dbusername, dbdrivername);
    // To construct kafka message.
    standbyParams.put("dbdrivername", dbdrivername);
    standbyParams.put("dburl", dburl);
    standbyParams.put("dbusername", dbusername);

    if (sqlStatement instanceof MySqlInsertStatement) {
        AutoCompensateInsertHandler.newInstance().prepareCompensationAfterInserting(delegate, sqlStatement, executeSql, globalTxId, localTxId, server, standbyParams);
    } else if (sqlStatement instanceof MySqlUpdateStatement) {
        AutoCompensateUpdateHandler.newInstance().prepareCompensationAfterUpdating(delegate, sqlStatement, executeSql, globalTxId, localTxId, server, standbyParams);
    }
}
 
Example #15
Source File: DruidMysqlCreateTableTest.java    From dble with GNU General Public License v2.0 5 votes vote down vote up
private boolean isInsertHasSlot(String sql) {
    MySqlStatementParser parser = new MySqlStatementParser(sql);
    MySqlInsertStatement insertStatement = (MySqlInsertStatement) parser.parseStatement();
    List<SQLExpr> cc = insertStatement.getColumns();
    for (SQLExpr sqlExpr : cc) {
        SQLIdentifierExpr c = (SQLIdentifierExpr) sqlExpr;
        if ("_slot".equalsIgnoreCase(c.getName()) && cc.size() == insertStatement.getValues().getValues().size())
            return true;
    }
    return false;
}
 
Example #16
Source File: DefaultSQLRewrite.java    From Zebra with Apache License 2.0 5 votes vote down vote up
@Override
protected void printValuesList(MySqlInsertStatement x) {
	print0(ucase ? "VALUES " : "values ");
	if (x.getValuesList().size() > 1) {
		incrementIndent();
	}

	if (this.batchInsertParamIndexes == null) { // original
		for (int i = 0; i < x.getValuesList().size(); ++i) {
			if (i != 0) {
				print(',');
				println();
			}
			x.getValuesList().get(i).accept(this);

		}
	} else { // batch insert
		int count = 0;
		for (int i = 0; i < x.getValuesList().size(); ++i) {
			if (this.batchInsertParamIndexes.contains(i)) {
				if (count > 0) {
					print(',');
					println();
				}
				x.getValuesList().get(i).accept(this);
				count++;
			}
		}
	}
	if (x.getValuesList().size() > 1) {
		decrementIndent();
	}
}
 
Example #17
Source File: InsertParser.java    From dts with Apache License 2.0 5 votes vote down vote up
@Override
public TableDataInfo getPresentValue(List<Object> sqlParamsList, MySqlInsertStatement parseSqlStatement,
    StatementAdapter statementAdapter, TableMetaInfo tableMetaInfo) throws SQLException {
    TableDataInfo txcTable = new TableDataInfo();
    txcTable.setTableName(parseSqlStatement.getTableName().getSimpleName());
    List<TxcLine> line = txcTable.getLine();
    List<SQLInsertStatement.ValuesClause> valuesList = parseSqlStatement.getValuesList();
    List<SQLExpr> columns = parseSqlStatement.getColumns();
    for (SQLInsertStatement.ValuesClause valuesClause : valuesList) {
        List<SQLExpr> values = valuesClause.getValues();
        TxcLine txcLine = new TxcLine();
        for (int i = 0; i < columns.size(); i++) {
            TxcField txcField = new TxcField();
            String columnName = SQLUtils.toSQLString(columns.get(i)).replace("\'", "").replace("`", "").trim();
            txcField.setName(columnName);
            if (sqlParamsList != null && !sqlParamsList.isEmpty()) {
                if (columnName.equalsIgnoreCase(tableMetaInfo.getAutoIncrementPrimaryKey())) {
                    sqlParamsList.add(i, getAutoIncrementPrimaryKeyValue(statementAdapter.getStatement()));
                }
                txcField.setValue(sqlParamsList.get(i));
            } else {
                txcField.setValue(SQLUtils.toSQLString(values.get(i)));
            }
            txcField.setJdkValue(SerializeUtils.serialize(txcField.getValue()));
            txcLine.getFields().add(txcField);
        }
        line.add(txcLine);
    }
    return txcTable;
}
 
Example #18
Source File: DruidInsertParser.java    From dble with GNU General Public License v2.0 4 votes vote down vote up
private String convertInsertSQL(SchemaInfo schemaInfo, MySqlInsertStatement insert, String originSql, TableConfig tc) throws SQLNonTransientException {

        TableMeta orgTbMeta = ProxyMeta.getInstance().getTmManager().getSyncTableMeta(schemaInfo.getSchema(), schemaInfo.getTable());
        if (orgTbMeta == null)
            return originSql;

        boolean isAutoIncrement = tc.isAutoIncrement();

        StringBuilder sb = new StringBuilder(200);
        sb.append("insert ");
        if (insert.isIgnore()) {
            sb.append("ignore ");
        }
        sb.append("into `");
        sb.append(schemaInfo.getTable());
        sb.append("`");

        List<SQLExpr> columns = insert.getColumns();

        int autoIncrement = -1;
        int colSize;
        // insert without columns :insert into t values(xxx,xxx)
        if (columns == null || columns.size() <= 0) {
            if (isAutoIncrement) {
                autoIncrement = getIncrementKeyIndex(schemaInfo, tc.getIncrementColumn());
            }
            colSize = orgTbMeta.getColumns().size();
        } else {
            genColumnNames(tc, isAutoIncrement, sb, columns);
            colSize = columns.size();
            if (isAutoIncrement) {
                getIncrementKeyIndex(schemaInfo, tc.getIncrementColumn());
                autoIncrement = columns.size();
                sb.append(",").append("`").append(tc.getIncrementColumn()).append("`");
                colSize++;
            }
            sb.append(")");
        }

        sb.append(" values");
        String tableKey = StringUtil.getFullName(schemaInfo.getSchema(), schemaInfo.getTable());
        List<ValuesClause> vcl = insert.getValuesList();
        if (vcl != null && vcl.size() > 1) { // batch insert
            for (int j = 0; j < vcl.size(); j++) {
                if (j != vcl.size() - 1)
                    appendValues(tableKey, vcl.get(j).getValues(), sb, autoIncrement, colSize).append(",");
                else
                    appendValues(tableKey, vcl.get(j).getValues(), sb, autoIncrement, colSize);
            }
        } else {
            List<SQLExpr> values = insert.getValues().getValues();
            appendValues(tableKey, values, sb, autoIncrement, colSize);
        }

        List<SQLExpr> dku = insert.getDuplicateKeyUpdate();
        if (dku != null && dku.size() > 0) {
            genDuplicate(sb, dku);
        }
        return RouterUtil.removeSchema(sb.toString(), schemaInfo.getSchema());
    }
 
Example #19
Source File: InsertParser.java    From dts with Apache License 2.0 4 votes vote down vote up
@Override
public TableDataInfo getOriginValue(List<Object> whereParamsList, MySqlInsertStatement parseSqlStatement,
    StatementAdapter statementAdapter, TableMetaInfo tableMetaInfo) {
    return null;
}
 
Example #20
Source File: InsertHandler.java    From dble with GNU General Public License v2.0 4 votes vote down vote up
@Override
public SQLStatement preHandle(DumpFileContext context, String stmt) throws DumpException, SQLNonTransientException {
    // get table name simply
    String table = null;
    Matcher matcher = InsertHandler.INSERT_STMT.matcher(stmt);
    if (matcher.find()) {
        table = matcher.group(1);
    }
    context.setTable(table);
    if (table != null && table.equalsIgnoreCase(currentTable)) {
        if (context.isSkipContext() || context.getTableType() == TableType.DEFAULT) {
            return null;
        }
        return RouteStrategyFactory.getRouteStrategy().parserSQL(stmt);
    } else {
        currentTable = table;
    }

    if (context.isSkipContext() || context.getTableType() == TableType.DEFAULT) {
        return null;
    }

    MySqlInsertStatement insert = (MySqlInsertStatement) RouteStrategyFactory.getRouteStrategy().parserSQL(stmt);
    // check columns from insert columns
    checkColumns(context, insert.getColumns());
    // add
    StringBuilder insertHeader = new StringBuilder("INSERT INTO ");
    insertHeader.append("`");
    insertHeader.append(context.getTable());
    insertHeader.append("`");
    if (!CollectionUtil.isEmpty(insert.getColumns())) {
        insertHeader.append(insert.getColumns().toString());
    }
    insertHeader.append(" VALUES");
    if (context.getTableType() == TableType.SHARDING) {
        shardingValuesHandler.reset();
        valuesHandler = shardingValuesHandler;
    } else {
        valuesHandler = defaultValuesHandler;
    }
    valuesHandler.setInsertHeader(insertHeader);
    return insert;
}
 
Example #21
Source File: DefaultShardRouter.java    From Zebra with Apache License 2.0 4 votes vote down vote up
private void buildBatchInsertSqls(ShardEvalResult shardResult, SQLParsedResult parseResult, String logicTable, RouterResult routerResult) {
	List<RouterTarget> sqls = new ArrayList<RouterTarget>();
	Map<String, Set<String>> dbAndTables = shardResult.getDbAndTables();
	Map<String, Map<String, Set<Integer>>> insertClauseIndexMap = shardResult.getInsertClauseIndexMap();

	MySqlInsertStatement stmt = (MySqlInsertStatement) parseResult.getStmt();
	int clauseColumnSize = stmt.getColumns().size();

	for (Entry<String, Set<String>> entry : dbAndTables.entrySet()) {
		String database = entry.getKey();
		RouterTarget targetedSql = new RouterTarget(database);
		Map<String, Set<Integer>> tbParamIndexMap = insertClauseIndexMap.get(database);

		int tableIndex = 0;
		for (String physicalTable : entry.getValue()) {
			Set<Integer> paramIndexes = null;
			if (tbParamIndexMap != null) {
				paramIndexes = tbParamIndexMap.get(physicalTable);
			}
			String newSql = sqlRewrite.rewrite(parseResult, logicTable, physicalTable, paramIndexes, null);

			String hintComment = parseResult.getRouterContext().getSqlhint().getHintComments();

			if (hintComment != null) {
				newSql = hintComment + newSql;
			}
			targetedSql.addSql(newSql);
			targetedSql.addPhysicalTable(physicalTable);

			List<Integer> mappingList = new ArrayList<Integer>();
			if (paramIndexes != null) {
				for (Integer index : paramIndexes) {
					for(int i = index * clauseColumnSize; i < (index + 1) * clauseColumnSize; ++i) {
						mappingList.add(i+1);
					}
				}
			}
			targetedSql.putParamIndexMappingList(tableIndex, mappingList);
			tableIndex++;
		}

		sqls.add(targetedSql);
	}

	routerResult.setSqls(sqls);
}
 
Example #22
Source File: CompensateService.java    From txle with Apache License 2.0 4 votes vote down vote up
public void prepareBackupSql(TxleTransactionStart tx, TxleTxStartAck.Builder txStartAck, boolean isExistsGlobalTx, Map<String, String> localTxBackupSql) {
        for (TxleSubTransactionStart subTx : tx.getSubTxInfoList()) {
            try {
                SQLStatement sqlStatement = new MySqlStatementParser(subTx.getSql()).parseStatement();
                String tableName = parseTableName(sqlStatement);
                String tableNameWithSchema = tableName;
                if (tableName.indexOf(".") < 0) {
                    tableNameWithSchema = subTx.getDbSchema() + "." + tableName;
                } else {
                    tableName = tableName.substring(tableName.indexOf(".") + 1);
                }
                String txleOldBackupTableName = TxleConstants.giveBackupTableNameForOldData(subTx.getDbSchema(), tableName), txleOldBackupTableNameWithSchema = this.schema + "." + txleOldBackupTableName;
                String txleNewBackupTableName = TxleConstants.giveBackupTableNameForNewData(subTx.getDbSchema(), tableName), txleNewBackupTableNameWithSchema = this.schema + "." + txleNewBackupTableName;

                // create backup table & alter structure
                TxleSubTxSql.Builder subTxSql = TxleSubTxSql.newBuilder().setLocalTxId(subTx.getLocalTxId()).setDbNodeId(subTx.getDbNodeId()).setDbSchema(subTx.getDbSchema()).setOrder(subTx.getOrder());
                this.constructBackupSqls(tx, isExistsGlobalTx, subTx, subTxSql, tableNameWithSchema, txleOldBackupTableName, txleOldBackupTableNameWithSchema, txleNewBackupTableName, txleNewBackupTableNameWithSchema);

                String operation = "";
                if (sqlStatement instanceof MySqlInsertStatement) {
                    operation = "insert";
                    // the formal business sql
                    subTxSql.addSubTxSql(subTx.getSql());
                    // the backup new data sql
                    Object primaryKey = this.txleEhCache.get(TxleCacheType.INIT, subTx.getDbNodeId() + "." + tableNameWithSchema);
                    if (primaryKey == null) {
                        primaryKey = "id";
                    }
                    subTxSql.addSubTxSql(String.format("INSERT INTO " + txleNewBackupTableNameWithSchema + " SELECT *, '%s', '%s' FROM %s WHERE " + primaryKey + " = (SELECT LAST_INSERT_ID()) FOR UPDATE " + TxleConstants.ACTION_SQL, tx.getGlobalTxId(), subTx.getLocalTxId(), tableNameWithSchema));
                } else if (sqlStatement instanceof MySqlDeleteStatement) {
                    operation = "delete";
                    subTxSql.addSubTxSql(String.format("INSERT INTO " + txleOldBackupTableNameWithSchema + " SELECT *, '%s', '%s' FROM %s WHERE %s FOR UPDATE "
                            + TxleConstants.ACTION_SQL, tx.getGlobalTxId(), subTx.getLocalTxId(), tableNameWithSchema, ((MySqlDeleteStatement) sqlStatement).getWhere().toString()));
                    subTxSql.addSubTxSql(subTx.getSql());
                } else if (sqlStatement instanceof MySqlUpdateStatement) {
                    operation = "update";
                    subTxSql.addSubTxSql(String.format("INSERT INTO " + txleOldBackupTableNameWithSchema + " SELECT *, '%s', '%s' FROM %s WHERE %s FOR UPDATE "
                            + TxleConstants.ACTION_SQL, tx.getGlobalTxId(), subTx.getLocalTxId(), tableNameWithSchema, ((MySqlUpdateStatement) sqlStatement).getWhere().toString()));
                    subTxSql.addSubTxSql(subTx.getSql());
                    subTxSql.addSubTxSql(String.format("INSERT INTO " + txleNewBackupTableNameWithSchema + " SELECT *, '%s', '%s' FROM %s WHERE %s FOR UPDATE "
                            + TxleConstants.ACTION_SQL, tx.getGlobalTxId(), subTx.getLocalTxId(), tableNameWithSchema, ((MySqlUpdateStatement) sqlStatement).getWhere().toString()));
                }

                if (!txleEhCache.readConfigCache(TxleConstants.getServiceInstanceId(tx.getServiceName(), tx.getServiceIP()), tx.getServiceCategory(), ConfigCenterType.ClientCompensate)) {
                    subTxSql.addSubTxSql("set autocommit=0");
                    subTxSql.addSubTxSql("commit");
                    subTxSql.addSubTxSql("set autocommit=1");
//                    subTxSql.addSubTxSql("begin");
                }

                // return SQLs to client
                txStartAck.addSubTxSql(subTxSql.build());
                final StringBuilder backupSqls = new StringBuilder();
                subTxSql.getSubTxSqlList().forEach(sql -> backupSqls.append(sql + TxleConstants.STRING_SEPARATOR));
                localTxBackupSql.put(subTx.getLocalTxId(), backupSqls.toString());

                kafkaMessageRepository.save(new KafkaMessage(tx.getGlobalTxId(), subTx.getLocalTxId(), "", subTx.getDbNodeId(), "", tableNameWithSchema, operation, ""));
            } catch (Exception e) {
                handleExceptionWithFaultToleranceChecking("Failed to prepare sqls for backup.", e, tx, txStartAck);
            }
        }
    }
 
Example #23
Source File: InsertParser.java    From dts with Apache License 2.0 4 votes vote down vote up
@Override
protected String getTableName(MySqlInsertStatement parseSqlStatement) {
    return parseSqlStatement.getTableName().getSimpleName();
}
 
Example #24
Source File: InsertParser.java    From dts with Apache License 2.0 4 votes vote down vote up
@Override
protected String selectSql(MySqlInsertStatement parseSqlStatement, Set<String> primaryKeyNameSet) {
    throw new UnsupportedOperationException(" do not support select sql for insert parser");
}
 
Example #25
Source File: PaserExecutor.java    From dts with Apache License 2.0 4 votes vote down vote up
public static SQLType parse(StatementAdapter txcStatement) throws SQLException {
    long start = System.currentTimeMillis();
    SQLType sqlType = SQLType.SELECT;
    try {
        DbRuntimeContext txcRuntimeContext = txcStatement.getConnection().getConnectionRuntimeContext();
        String sql = txcStatement.getSql();
        SQLStatement sqlParseStatement = new MySqlStatementParser(sql).parseStatement();
        CommitInfo commitInfo = null;
        if (sqlParseStatement instanceof MySqlUpdateStatement) {
            commitInfo = UpdateParser.getInstance().parse(txcStatement);
            sqlType = SQLType.UPDATE;
            if (!Objects.isNull(commitInfo)&&!CollectionUtils.isEmpty(commitInfo.getOriginalValue().getLine())) {
                txcRuntimeContext.getInfo().add(commitInfo);
                fillDbMetaAndLockRow(txcStatement, commitInfo);
            }
        } else if (sqlParseStatement instanceof MySqlInsertStatement) {
            sqlType = SQLType.INSERT;
        } else if (sqlParseStatement instanceof MySqlDeleteStatement) {
            commitInfo = DeleteParser.getInstance().parse(txcStatement);
            sqlType = SQLType.DELETE;
            if (!Objects.isNull(commitInfo) && !CollectionUtils.isEmpty(commitInfo.getOriginalValue().getLine())) {
                txcRuntimeContext.getInfo().add(commitInfo);
                fillDbMetaAndLockRow(txcStatement, commitInfo);
            }
        } else if (sqlParseStatement instanceof SQLSelectStatement) {
            SQLSelectQueryBlock selectQueryBlock =
                ((SQLSelectStatement)sqlParseStatement).getSelect().getQueryBlock();
            if (selectQueryBlock.getFrom() != null) {
                SelectParser.getInstance().parse(txcStatement);
                sqlType = SQLType.SELECT;
            }
        }
    } catch (Exception e) {
        logger.error("parse sql error", e);
        if (e instanceof SQLException || e instanceof RuntimeException) {
            throw e;
        } else {
            throw new SQLException(e);
        }
    } finally {
        long cost = System.currentTimeMillis() - start;
        if (sqlType != SQLType.SELECT || cost > 50) {
            logger.debug("parser sql:{}, cost:{}ms", txcStatement.getSql(), cost);
        }
    }
    return sqlType;
}
 
Example #26
Source File: AutoCompensateService.java    From txle with Apache License 2.0 4 votes vote down vote up
private boolean checkDataConsistency(String compensateSql, String globalTxId, String localTxId) throws NoSuchAlgorithmException, IOException {
    MySqlStatementParser parser = new MySqlStatementParser(compensateSql);
    SQLStatement sqlStatement = parser.parseStatement();
    if (sqlStatement instanceof MySqlUpdateStatement ||  sqlStatement instanceof MySqlInsertStatement) {
        MySqlUpdateStatement deleteStatement = (MySqlUpdateStatement) sqlStatement;
        String tableName = deleteStatement.getTableName().toString().toLowerCase();
        String schema = TxleConstants.APP_NAME;
        String txleBackupTableName = "backup_new_" + tableName;
        int backupDataCount = autoCompensateDao.executeQueryCount("SELECT count(*) FROM " + schema + "." + txleBackupTableName + " T WHERE T.globalTxId = ? AND T.localTxId = ? FOR UPDATE", globalTxId, localTxId);
        if (backupDataCount > 0) {
            String pkName = this.parsePrimaryKeyColumnName(tableName);
            int currentDataCount = autoCompensateDao.executeQueryCount("SELECT count(*) FROM " + tableName + " T WHERE T.id IN (SELECT T1.id FROM " + schema + "." + txleBackupTableName + " T1 WHERE T1.globalTxId = ? AND T1.localTxId = ?)", globalTxId, localTxId);
            // in case of updating many times for some same data, to delete the previous changes, so it only has one backup for any data.
            if (backupDataCount == currentDataCount) {
                List<Map<String, Object>> columnList = autoCompensateDao.executeQuery(
                        "SELECT GROUP_CONCAT(COLUMN_NAME) COLUMN_NAMES FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + schema + "' AND TABLE_NAME = '" + txleBackupTableName + "' AND COLUMN_NAME NOT IN ('globalTxId', 'localTxId')");
                if (columnList != null && !columnList.isEmpty()) {
                    StringBuilder columnNames = new StringBuilder();
                    String[] columnArr = columnList.get(0).get("COLUMN_NAMES").toString().split(",");
                    for (String column : columnArr) {
                        if (columnNames.length() == 0) {
                            columnNames.append("T." + column);
                        } else {
                            columnNames.append(",T." + column);
                        }
                    }
                    String backupDataSql = "SELECT " + columnNames + " FROM " + schema + "." + txleBackupTableName + " T WHERE T.globalTxId = '" + globalTxId + "' AND T.localTxId = '" + localTxId + "'";
                    String currentDataSql = "SELECT " + columnNames + " FROM " + tableName + " T, " + schema + "." + txleBackupTableName + " T1 WHERE T." + pkName + " = T1." + pkName + " AND T1.globalTxId = '" + globalTxId + "' AND T1.localTxId = '" + localTxId + "'";

                    String backupDataMD5 = getMD5Digest(backupDataSql, backupDataCount);
                    String currentDataMD5 = getMD5Digest(currentDataSql, currentDataCount);
                    if (backupDataMD5.equals(currentDataMD5)) {
                        return true;
                    }
                }
            }
        }
        throw new RuntimeException("That's not consistent between backup data and current data.");
    }
    return true;
}
 
Example #27
Source File: AutoCompensateHandler.java    From txle with Apache License 2.0 4 votes vote down vote up
@Override
public void prepareCompensationBeforeExecuting(PreparedStatement delegate, String executeSql, Map<String, Object> standbyParams) throws SQLException {
    String globalTxId = CurrentThreadOmegaContext.getGlobalTxIdFromCurThread();
    if (globalTxId == null || globalTxId.length() == 0) {
        return;
    }
    String localTxId = CurrentThreadOmegaContext.getLocalTxIdFromCurThread();
    if (localTxId == null || localTxId.length() == 0) {
        return;
    }

    // To parse SQL by SQLParser tools from Druid.
    MySqlStatementParser parser = new MySqlStatementParser(executeSql);
    SQLStatement sqlStatement = parser.parseStatement();
    if (sqlStatement instanceof MySqlSelectIntoStatement) {
        return;
    }

    if (standbyParams == null) {
        standbyParams = new HashMap<>(8);
    }

    String server = CurrentThreadOmegaContext.getServiceNameFromCurThread();

    // To set a relationship between localTxId and datSourceInfo, in order to determine to use the relative dataSource for localTxId when it need be compensated.
    DatabaseMetaData databaseMetaData = delegate.getConnection().getMetaData();
    String dburl = databaseMetaData.getURL(), dbusername = databaseMetaData.getUserName(), dbdrivername = databaseMetaData.getDriverName();
    DataSourceMappingCache.putLocalTxIdAndDataSourceInfo(localTxId, dburl, dbusername, dbdrivername);
    // To construct kafka message.
    standbyParams.put("dbdrivername", dbdrivername);
    standbyParams.put("dburl", dburl);
    standbyParams.put("dbusername", dbusername);

    if (sqlStatement instanceof MySqlInsertStatement) {
        return;
    } else if (sqlStatement instanceof MySqlUpdateStatement) {
        AutoCompensateUpdateHandler.newInstance().prepareCompensationBeforeUpdating(delegate, sqlStatement, executeSql, globalTxId, localTxId, server, standbyParams);
    } else if (sqlStatement instanceof MySqlDeleteStatement) {
        AutoCompensateDeleteHandler.newInstance().prepareCompensationBeforeDeleting(delegate, sqlStatement, executeSql, globalTxId, localTxId, server, standbyParams);
    } else {
        standbyParams.clear();
        // Default is closed, means that just does record, if it's open, then program will throw an exception about current special SQL, just for auto-compensation.
        boolean checkSpecialSql = TxleStaticConfig.getBooleanConfig("txle.transaction.auto-compensation.check-special-sql", false);
        if (checkSpecialSql) {
            throw new SQLException(TxleConstants.logErrorPrefixWithTime() + "Do not support sql [" + executeSql + "] to auto-compensation.");
        } else {
            LOG.debug(TxleConstants.logDebugPrefixWithTime() + "Do not support sql [{}] to auto-compensation, but it has been executed due to the switch 'checkSpecialSql' is closed.", executeSql);
        }
    }
}
 
Example #28
Source File: DruidMycatRouteStrategy.java    From Mycat2 with GNU General Public License v3.0 4 votes vote down vote up
/**
 *  直接结果路由
 * @param rrs
 * @param ctx
 * @param schema
 * @param druidParser
 * @param statement
 * @param cachePool
 * @return
 * @throws SQLNonTransientException
 */
private RouteResultset directRoute(RouteResultset rrs,DruidShardingParseInfo ctx,SchemaConfig schema,
		DruidParser druidParser,SQLStatement statement,LayerCachePool cachePool) throws SQLNonTransientException{

	//改写sql:如insert语句主键自增长, 在直接结果路由的情况下,进行sql 改写处理
	druidParser.changeSql(schema, rrs, statement,cachePool);

	/**
	 * DruidParser 解析过程中已完成了路由的直接返回
	 */
	if ( rrs.isFinishedRoute() ) {
		return rrs;
	}

	/**
	 * 没有from的select语句或其他
	 */
	if((ctx.getTables() == null || ctx.getTables().size() == 0)&&(ctx.getTableAliasMap()==null||ctx.getTableAliasMap().isEmpty()))
	{
		return RouterUtil.routeToSingleNode(rrs, schema.getRandomDataNode(), druidParser.getCtx().getSql());
	}

	if(druidParser.getCtx().getRouteCalculateUnits().size() == 0) {
		RouteCalculateUnit routeCalculateUnit = new RouteCalculateUnit();
		druidParser.getCtx().addRouteCalculateUnit(routeCalculateUnit);
	}

	SortedSet<RouteResultsetNode> nodeSet = new TreeSet<RouteResultsetNode>();
	boolean isAllGlobalTable = RouterUtil.isAllGlobalTable(ctx, schema);
	for(RouteCalculateUnit unit: druidParser.getCtx().getRouteCalculateUnits()) {
		RouteResultset rrsTmp = RouterUtil.tryRouteForTables(schema, druidParser.getCtx(), unit, rrs, isSelect(statement), cachePool);
		if(rrsTmp != null&&rrsTmp.getNodes()!=null) {
			for(RouteResultsetNode node :rrsTmp.getNodes()) {
				nodeSet.add(node);
			}
		}
		if(isAllGlobalTable) {//都是全局表时只计算一遍路由
			break;
		}
	}

	RouteResultsetNode[] nodes = new RouteResultsetNode[nodeSet.size()];
	int i = 0;
	for (RouteResultsetNode aNodeSet : nodeSet) {
		nodes[i] = aNodeSet;
		if(statement instanceof MySqlInsertStatement &&ctx.getTables().size()==1&&schema.getTables().containsKey(ctx.getTables().get(0))) {
			RuleConfig rule = schema.getTables().get(ctx.getTables().get(0)).getRule();
			if(rule!=null&&  rule.getRuleAlgorithm() instanceof SlotFunction){
				aNodeSet.setStatement(ParseUtil.changeInsertAddSlot(aNodeSet.getStatement(),aNodeSet.getSlot()));
			}
		}
		i++;
	}
	rrs.setNodes(nodes);

	//分表
	/**
	 *  subTables="t_order$1-2,t_order3"
	 *目前分表 1.6 开始支持 幵丏 dataNode 在分表条件下只能配置一个,分表条件下不支持join。
	 */
	if(rrs.isDistTable()){
		return this.routeDisTable(statement,rrs);
	}
	return rrs;
}
 
Example #29
Source File: CompensateService.java    From txle with Apache License 2.0 4 votes vote down vote up
public void constructCompensateSql(TxleTransactionStart tx, TxleTxStartAck.Builder txStartAck, Map<String, String> localTxCompensateSql) {
    for (TxleSubTransactionStart subTx : tx.getSubTxInfoList()) {
        try {
            SQLStatement sqlStatement = new MySqlStatementParser(subTx.getSql()).parseStatement();
            String tableName = parseTableName(sqlStatement);
            String tableNameWithSchema = tableName;
            if (tableName.indexOf(".") < 0) {
                tableNameWithSchema = subTx.getDbSchema() + "." + tableName;
            } else {
                tableName = tableName.substring(tableName.indexOf(".") + 1);
            }
            String txleOldBackupTableName = TxleConstants.giveBackupTableNameForOldData(subTx.getDbSchema(), tableName);
            String txleNewBackupTableName = this.schema + "." + TxleConstants.giveBackupTableNameForNewData(subTx.getDbSchema(), tableName);

            String compensateSql = "";
            if (sqlStatement instanceof MySqlInsertStatement) {
                compensateSql = String.format("DELETE FROM " + tableNameWithSchema + " WHERE id IN (SELECT id FROM " + txleNewBackupTableName + " WHERE globalTxId = '%s' AND localTxId = '%s') " + TxleConstants.ACTION_SQL, tx.getGlobalTxId(), subTx.getLocalTxId());
            } else if (sqlStatement instanceof MySqlDeleteStatement) {
                compensateSql = String.format("INSERT INTO " + tableNameWithSchema + " SELECT %s FROM " + schema + "." + txleOldBackupTableName + " WHERE globalTxId = '%s' AND localTxId = '%s' "
                        + TxleConstants.ACTION_SQL, this.readColumnNames(subTx.getDbSchema(), tableName), tx.getGlobalTxId(), subTx.getLocalTxId());
            } else if (sqlStatement instanceof MySqlUpdateStatement) {
                String setColumns = this.constructSetColumnsForUpdate(subTx.getDbSchema(), tableName);
                Object primaryKey = this.txleEhCache.get(TxleCacheType.INIT, subTx.getDbNodeId() + "." + tableNameWithSchema);
                if (primaryKey == null) {
                    List list = customRepository.executeQuery("SELECT T.field FROM BusinessDBLatestDetail T WHERE T.isprimarykey = 1 AND T.node = ? AND T.dbschema = ? AND T.tablename = ?", subTx.getDbNodeId(), subTx.getDbSchema(), tableName);
                    if (list != null && !list.isEmpty()) {
                        primaryKey = list.get(0);
                        if (primaryKey != null) {
                            txleEhCache.put(TxleCacheType.INIT, subTx.getDbNodeId() + "." + subTx.getDbSchema() + "." + tableName, primaryKey);
                        }
                    }
                }
                if (primaryKey == null) {
                    primaryKey = "id";
                }
                // construct reversed sql
                compensateSql = String.format("UPDATE %s T INNER JOIN %s T1 ON T." + primaryKey + " = T1." + primaryKey + " SET %s WHERE T1.globalTxId = '%s' AND T1.localTxId = '%s' "
                        + TxleConstants.ACTION_SQL, tableNameWithSchema, this.schema + "." + txleOldBackupTableName, setColumns, tx.getGlobalTxId(), subTx.getLocalTxId());
            }
            localTxCompensateSql.put(subTx.getLocalTxId(), compensateSql);
        } catch (Exception e) {
            handleExceptionWithFaultToleranceChecking("Failed to construct sql for compensation.", e, tx, txStartAck);
        }
    }
}
 
Example #30
Source File: DruidInsertParser.java    From Mycat2 with GNU General Public License v3.0 4 votes vote down vote up
/**
	 * 考虑因素:isChildTable、批量、是否分片
	 */
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
		MySqlInsertStatement insert = (MySqlInsertStatement)stmt;
		String tableName = StringUtil.removeBackquote(insert.getTableName().getSimpleName()).toUpperCase();

		ctx.addTable(tableName);
		if(RouterUtil.isNoSharding(schema,tableName)) {//整个schema都不分库或者该表不拆分
			RouterUtil.routeForTableMeta(rrs, schema, tableName, rrs.getStatement());
			rrs.setFinishedRoute(true);
			return;
		}

		TableConfig tc = schema.getTables().get(tableName);
		if(tc == null) {
			String msg = "can't find table define in schema "
					+ tableName + " schema:" + schema.getName();
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		} else {
			//childTable的insert直接在解析过程中完成路由
			if (tc.isChildTable()) {
				parserChildTable(schema, rrs, tableName, insert);
				return;
			}
			
			String partitionColumn = tc.getPartitionColumn();
			
			if(partitionColumn != null) {//分片表
				//拆分表必须给出column list,否则无法寻找分片字段的值
				if(insert.getColumns() == null || insert.getColumns().size() == 0) {
					throw new SQLSyntaxErrorException("partition table, insert must provide ColumnList");
				}
				
				//批量insert
				if(isMultiInsert(insert)) {
//					String msg = "multi insert not provided" ;
//					LOGGER.warn(msg);
//					throw new SQLNonTransientException(msg);
					parserBatchInsert(schema, rrs, partitionColumn, tableName, insert);
				} else {
					parserSingleInsert(schema, rrs, partitionColumn, tableName, insert);
				}
				
			}
		}
	}