NADISH REDDY

Description: This project automates loading Delta Cargo Customers from Source System to Salesforce CRM using Orchestration Framework and Dedicated Azure Data Factory. It involves automated data extraction from Source Systems, and staging in Centralized Curated Zone before sending it to Salesforce. The process ensures efficient and reliable data integration, providing a complete view of both Cargo and Cabin customer needs. Responsibilities: • Design and implement the data extraction and integration workflow using Azure Data Factory, Azure Data Lake Storage, Azure Data bricks, and Azure Functions. • Develop and implement the Orchestration Framework to schedule and trigger the ETL process at scheduled intervals. • Clean, enrich, and transform the data using Azure Databricks to ensure data quality and consistency. • Prepare and write the data to Salesforce staging objects using Azure Data Factory. • Implement monitoring and logging mechanisms to track the data extraction and integration process. • Collaborate with stakeholders, including the Delta Cargo team and Salesforce team, to ensure that data integration meets organizational data needs and requirements. • Troubleshoot and resolve any issues or errors that may arise during the data extraction and integration process, ensuring the smooth and reliable operation of the workflow. • Create and maintain documentation, including technical specifications, design documents, and operational procedures. • Collaborate with other team members, such as data analysts, data scientists, and business stakeholders, to understand data requirements and ensure data accuracy and integrity. • Stay up to date with the latest technologies and best practices in data engineering and integration and recommend improvements or optimizations to the workflow as needed. • Conduct performance tuning and optimization of the data extraction and integration process to ensure efficient and timely data processing.

Description: This project involves an end-to-end ETL process to collect, transform, and load data from various sources into the Snowflake data warehouse for efficient reporting and analytics. ADF is used to ingest raw data into ADLS Gen-2, which is then cleansed and transformed using Snowflake Database components. Snow SQL scripts are used to populate data into Dimension & Fact tables in Snowflake, including inserting, updating, or deleting data. The project also includes creating reporting views that simplify access to data for reporting and analytics. Finally, ADF is used for the orchestration, scheduling, and triggering of pipelines, ensuring a seamless and error-free ETL process. The process valuable insights for decision-making. Responsibilities: • Developing ADF pipelines to extract data from source systems and FTP servers, and ingest raw files into BLOB storage in ADLS Gen-2. • Designing and developing Snowflake Database components, including creating tables, schemas, views, and materialized views, based on the business requirements. • Creating ADF pipelines to load data into Snowflake for Full load & Incremental Loads, ensuring data accuracy and efficiency. • Developing Snow SQL scripts based on the mapping rules to populate the data into Dimension & Fact tables in Snowflake, and ensuring data integrity and performance optimization. • Develop reporting views based on user requirements, using Snowflake's capabilities for creating views and materialized views, to provide a simplified and optimized way to access data for reporting and analytics. • Using ADF to orchestrate the entire ETL pipelines, ensuring data processing in the right sequence and with proper error handling, and scheduling or triggering pipelines based on business requirements and data availability. • Collaborating with cross-functional teams, such as data scientists, data analysts, and business stakeholders, to understand their requirements and deliver data solutions that meet their needs.

Description: Société Générale has established a collaborative network with various stakeholders to deliver a seamless banking experience to customers by leveraging advanced technologies like Big Data and AI. The API platform has created innovative and customer-centric experiences, with well-defined regulatory controls, privacy, and data integrity, available for use by other brands, fintech's, and software developers to plug into their applications. This has accelerated the bank's digital channels penetration and access to customer lifestyle. Responsibilities: •Developed data pipelines using IBM CDC and Kafka with Spark direct streaming, ensuring efficient and reliable data ingestion and processing. •Created Spark scripts using Scala shell commands based on project requirements, and optimized existing algorithms in Hadoop using Spark Context, Spark-SQL, Data Frames, and Pair RDDs. •Performed advanced procedures like text analytics and processing using the in-memory computing capabilities of Spark, enabling insights from unstructured data. •Experienced in handling large datasets using techniques such as Partitions, Spark in Memory capabilities, Broadcasts in Spark, and effective & efficient Joins, Transformations, and other techniques during the ingestion process itself, ensuring optimal performance and scalability. •Worked with Sqoop for importing metadata from Oracle, ensuring smooth and efficient data ingestion from relational databases. •Involved in creating Hive tables, loading data, and analyzing data using Hive queries, enabling data exploration and visualization. •Developed Hive queries to process the data and generate data cubes for visualization, providing valuable insights for business stakeholders and Tableau to generate daily reports of data. •Implemented schema extraction for Parquet, ORC, and Avro file formats in Hive, ensuring proper data organization and optimized query performance. •Implemented Partitioning, Dynamic Partitions, and Buckets in HIVE.

Description: • The service is designed for large insurance providers to help them manage a network of facilities by giving insights into facilities’ financial state and operation, highlighting trends, and generating forecasts. The service securely analyzes patient data, payments, insurance programs, creating in-depth reports that assess Client satisfaction rate, the quality of the services provided, optional services provided, and general performance of a company, which in turn helps to develop effective business strategy. This Analytics service is developed in Scala and relies on Kafka, Spark, and Casandra database to implement the big data processing. Kafka is used to collect and cache data provided by a multitude of client applications. Spark consumes this data and stores it to Cassandra database, then Spark performs further data analysis and prepares reports that are also being stored to Cassandra. This data is used by a client application and an API that can be accessed by Insurance providers. Technologies: Hadoop, spark, Data Warehouse, PLSQL, SQL, Java, UNIX shell scripting Responsibilities: • Involving in Business Process analysis, Requirements review and identification of business impact. • Loading data from source system to Data warehousing by Migration. • Making user adhoc Reports and analyzing the big data environment. Worked in Map reduce and Hive, Spark environment. • Worked extensively with Sqoop for importing metadata from Oracle. • Involved in creating Hive tables, and loading and analyzing data using hive queries • Implemented Partitioning, Dynamic Partitions, Buckets in HIVE. • Used Reporting tools like Tableau to connect with Hive for generating daily reports of data. • Writing Programs for reports using map reduce, Hive, Spark SQL when required. • Handled user web logs using Kafka by creating Topics and analyzed using spark streaming • Co-ordinate defect status review meetings with the testing teams on issues with the defects in the plate.